diff --git a/pyproject.toml b/pyproject.toml
index e53d8ef..b09f128 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,7 +33,7 @@ classifiers = [
 [project.optional-dependencies]
 JOURNAL = ["systemd-python"]
 NOTIFICATIONS = ["dbus-python"]
-REMOTE = ["paramiko"]
+REMOTE = ["paramiko>=3.2"]
 
 [project.urls]
 Homepage = "https://git.shouldnt.work/fuxino/simple_backup"
diff --git a/simple_backup/simple_backup.py b/simple_backup/simple_backup.py
index 05bf67f..36b3833 100755
--- a/simple_backup/simple_backup.py
+++ b/simple_backup/simple_backup.py
@@ -36,7 +36,6 @@ from glob import glob
 
 try:
     import paramiko
-    from paramiko import RSAKey, Ed25519Key, ECDSAKey, DSSKey
 except ImportError:
     pass
 
@@ -397,48 +396,10 @@ class Backup:
 
                 return None
 
-        pkey = None
-
-        try:
-            pkey = RSAKey.from_private_key_file(self.ssh_keyfile)
-        except paramiko.PasswordRequiredException:
-            password = getpass(
-                f'Enter passwphrase for key \'{self.ssh_keyfile}\': ')
-
-            try:
-                pkey = RSAKey.from_private_key_file(self.ssh_keyfile, password)
-            except paramiko.SSHException:
-                pass
+        pkey = self._load_ssh_key()
 
         if pkey is None:
-            try:
-                pkey = Ed25519Key.from_private_key_file(self.ssh_keyfile)
-            except paramiko.PasswordRequiredException:
-                try:
-                    pkey = Ed25519Key.from_private_key_file(
-                        self.ssh_keyfile, password)
-                except paramiko.SSHException:
-                    pass
-
-        if pkey is None:
-            try:
-                pkey = ECDSAKey.from_private_key_file(self.ssh_keyfile)
-            except paramiko.PasswordRequiredException:
-                try:
-                    pkey = ECDSAKey.from_private_key_file(
-                        self.ssh_keyfile, password)
-                except paramiko.SSHException:
-                    pass
-
-        if pkey is None:
-            try:
-                pkey = DSSKey.from_private_key_file(self.ssh_keyfile)
-            except paramiko.PasswordRequiredException:
-                try:
-                    pkey = DSSKey.from_private_key_file(
-                        self.ssh_keyfile, password)
-                except paramiko.SSHException:
-                    pass
+            return None
 
         try:
             ssh.connect(self.ssh_host, username=self.ssh_user, pkey=pkey)
@@ -449,6 +410,37 @@ class Backup:
 
         return ssh
 
+    def _load_ssh_key(self) -> Optional[paramiko.PKey]:
+        try:
+            return paramiko.PKey.from_path(self.ssh_keyfile)
+        except TypeError:
+            # Raised when the key is encrypted and no passphrase was given
+            pass
+        except (OSError, ValueError, paramiko.UnknownKeyType) as e:
+            logger.critical('Unable to load SSH key %s', self.ssh_keyfile)
+            logger.critical(e)
+
+            return None
+
+        try:
+            # Fail instead of reading the passphrase with echo if no terminal is available
+            with warnings.catch_warnings():
+                warnings.simplefilter('error', GetPassWarning)
+                passphrase = getpass(
+                    f'Enter passphrase for key \'{self.ssh_keyfile}\': ')
+        except GetPassWarning as e:
+            logger.critical('Unable to get passphrase')
+            logger.critical(e)
+
+            return None
+
+        try:
+            return paramiko.PKey.from_path(self.ssh_keyfile, passphrase.encode())
+        except (ValueError, paramiko.SSHException):
+            logger.critical('Unable to load SSH key %s, wrong passphrase?', self.ssh_keyfile)
+
+            return None
+
     def _returncode_log(self, returncode: int) -> None:
         match returncode:
             case 2: