Base the progress bar on file counts

With --link-dest unchanged files are linked, not copied, so rsync's byte
percentage stayed near zero and the ETA was hours. Use the checked/total
file counts instead, and show no percentage or ETA while still scanning.

simple_backup/simple_backup.py

@@ -75,7 +75,7 @@ R = TypeVar('R')
 SSH_TIMEOUT = 15
 
 # An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
-PROGRESS_RE = re.compile(r'\s*(\S+)\s+(\d+)%\s+(\S+)\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=\d+/\d+\))?\s*')
+PROGRESS_RE = re.compile(r'\s*(\S+)\s+\d+%\s+\S+\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=(\d+)/(\d+)\))?\s*')
 PROGRESS_BAR_WIDTH = 30
 
 
@@ -111,6 +111,8 @@ class _ProgressBar:
         self._start = default_timer()
         # rsync reports ir-chk while incremental recursion is still finding files
         self._scanning = True
+        self._checked = 0
+        self._total = 0
         self._shown = False
 
     def update(self, record: str) -> bool:
@@ -121,25 +123,30 @@ class _ProgressBar:
         if m is None:
             return False
 
-        size, percent, speed, chk = m.group(1), int(m.group(2)), m.group(3), m.group(4)
+        size, chk = m.group(1), m.group(2)
 
+        # Records in the middle of a file have no counts, so keep the last ones
         if chk is not None:
             self._scanning = chk == 'ir'
+            self._total = int(m.group(4))
+            self._checked = self._total - int(m.group(3))
 
-        # rsync's own time field is the elapsed time on records ending a file, so estimate the ETA here
+        # Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out
-        if 0 < percent < 100:
+        if self._scanning or self._total == 0:
-            eta = int((default_timer() - self._start) * (100 - percent) / percent)
+            text = f'{self._checked:,} files checked  {size} copied  (scanning)'
-            text = f'{percent:3d}%  {size}  {speed}  ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
+            percent = None
         else:
-            text = f'{percent:3d}%  {size}  {speed}'
+            percent = self._checked * 100 // self._total
+            text = f'{percent:3d}%  {self._checked:,}/{self._total:,} files  {size} copied'
 
-        if self._scanning:
+            if 0 < percent < 100:
-            text += '  (scanning)'
+                eta = int((default_timer() - self._start) * (self._total - self._checked) / self._checked)
+                text += f'  ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
 
         width = get_terminal_size().columns - 1
 
-        # Fixed width, so the bar doesn't jump when the text changes length
+        # Fixed width, so the bar doesn't jump when the text changes length; no bar until the total is known
-        if len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
+        if percent is not None and len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
             filled = PROGRESS_BAR_WIDTH * percent // 100
             text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}]  {text}'
 

tests/test_local.py

@@ -237,18 +237,28 @@ class TestProgressBar:
         # 30 s elapsed at each update
         monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
 
+    def test_scanning(self, capsys: pytest.CaptureFixture[str]) -> None:
+        bar = sb._ProgressBar()
+
+        assert bar.update('          2.00M  25%    1.47MB/s    0:00:02 (xfr#1, ir-chk=1009/2632)')
+        assert capsys.readouterr().err == '\r1,623 files checked  2.00M copied  (scanning)\033[K'
+
     def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
         bar = sb._ProgressBar()
 
-        assert bar.update('          2.00M  25%    1.47MB/s    0:00:02 (xfr#1, ir-chk=2/4)')
+        # With --link-dest rsync's percentage leaves out linked files, so the files are counted instead
+        assert bar.update('      4,000,000   1%    2.97MB/s    0:00:01 (xfr#2, to-chk=1/4)')
         assert capsys.readouterr().err == \
-            '\r[#######-----------------------]   25%  2.00M  1.47MB/s  ETA 0:01:30  (scanning)\033[K'
+            '\r[######################--------]   75%  3/4 files  4,000,000 copied  ETA 0:00:10\033[K'
 
-    def test_without_human_readable(self, capsys: pytest.CaptureFixture[str]) -> None:
+    def test_keeps_counts_within_file(self, monkeypatch: pytest.MonkeyPatch,
+                                      capsys: pytest.CaptureFixture[str]) -> None:
+        monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0, 30.0]).__next__)
         bar = sb._ProgressBar()
 
-        assert bar.update('      4,000,000  50%    2.97MB/s    0:00:01 (xfr#2, to-chk=1/4)')
+        assert bar.update('          2.00M   1%    1.47MB/s    0:00:02 (xfr#1, to-chk=2/4)')
-        assert capsys.readouterr().err.endswith(']   50%  4,000,000  2.97MB/s  ETA 0:00:30\033[K')
+        assert bar.update('          3.00M   2%    1.47MB/s    0:00:03')
+        assert capsys.readouterr().err.endswith(']   50%  2/4 files  3.00M copied  ETA 0:00:30\033[K')
 
     def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None:
         bar = sb._ProgressBar()
@@ -264,7 +274,7 @@ class TestProgressBar:
 
         assert sb._ProgressBar().update('  6.00M 100%    1.48MB/s    0:00:03 (xfr#3, to-chk=0/4)')
         # No room for the bar
-        assert capsys.readouterr().err == '\r100%  6.00M  1.48MB\033[K'
+        assert capsys.readouterr().err == '\r100%  4/4 files  6.\033[K'
 
 
 class FakePopen: