diff --git a/simple_backup/simple_backup.py b/simple_backup/simple_backup.py
index 9e4d7f2..6e30a9a 100755
--- a/simple_backup/simple_backup.py
+++ b/simple_backup/simple_backup.py
@@ -295,9 +295,15 @@ class Backup:
                         count += 1
         else:
             try:
-                dirs = os.listdir(f'{self.output}/simple_backup')
+                # Only count directories, so that stray files don't count as backups
+                dirs = [f.name for f in os.scandir(f'{self.output}/simple_backup')
+                        if f.is_dir(follow_symlinks=False)]
             except FileNotFoundError:
                 return
+            except PermissionError:
+                logger.error('Cannot list old backups. Permission denied. Old backups will not be removed')
+
+                return
 
             n_backup = len(dirs)