Add a progress bar for rsync

On a terminal, run rsync with --info=progress2 and draw its updates as a
progress bar on stderr. The ETA is estimated locally, since rsync reports
the elapsed time when a file finishes. Disable it with --no-progress.

man/simple_backup.1

@@ -105,6 +105,12 @@ according to \-\-keep will be left, and the new backup may be incomplete. In par
 .B \-\-no\-syslog
 Don't use systemd journal for logging.
 .TP
+.B \-\-no\-progress
+Don't show a progress bar while rsync copies the files. The progress bar is only shown when
+the output is a terminal, so it never appears in the systemd journal or in redirected output.
+While rsync is still looking for files, the bar shows \(aq(scanning)\(aq, and the percentage
+and estimated time are only approximate.
+.TP
 .B \-\-rsync\-options OPTIONS [OPTION...]
 By default, the following rsync options are used:
 .RS

simple_backup/simple_backup.py

@@ -23,7 +23,7 @@ import re
 from typing import Callable, Iterator, List, Optional, ParamSpec, TypedDict, TypeVar
 import warnings
 from functools import wraps
-from shutil import rmtree, which
+from shutil import get_terminal_size, rmtree, which
 import shlex
 import argparse
 import configparser
@@ -74,6 +74,10 @@ R = TypeVar('R')
 # Seconds to wait for the TCP connection to the SSH server
 SSH_TIMEOUT = 15
 
+# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
+PROGRESS_RE = re.compile(r'\s*(\S+)\s+(\d+)%\s+(\S+)\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=\d+/\d+\))?\s*')
+PROGRESS_BAR_WIDTH = 30
+
 
 def timing(func: Callable[P, R]) -> Callable[P, R]:
     """Decorator to measure execution time of a function
@@ -100,6 +104,59 @@ class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFo
     """Custom format for argparse help text"""
 
 
+class _ProgressBar:
+    """Draws rsync --info=progress2 updates as a bar on stderr"""
+
+    def __init__(self) -> None:
+        self._start = default_timer()
+        # rsync reports ir-chk while incremental recursion is still finding files
+        self._scanning = True
+        self._shown = False
+
+    def update(self, record: str) -> bool:
+        """Draw the bar if record is a progress update, and return whether it was"""
+
+        m = PROGRESS_RE.fullmatch(record)
+
+        if m is None:
+            return False
+
+        size, percent, speed, chk = m.group(1), int(m.group(2)), m.group(3), m.group(4)
+
+        if chk is not None:
+            self._scanning = chk == 'ir'
+
+        # rsync's own time field is the elapsed time on records ending a file, so estimate the ETA here
+        if 0 < percent < 100:
+            eta = int((default_timer() - self._start) * (100 - percent) / percent)
+            text = f'{percent:3d}%  {size}  {speed}  ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
+        else:
+            text = f'{percent:3d}%  {size}  {speed}'
+
+        if self._scanning:
+            text += '  (scanning)'
+
+        width = get_terminal_size().columns - 1
+
+        # Fixed width, so the bar doesn't jump when the text changes length
+        if len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
+            filled = PROGRESS_BAR_WIDTH * percent // 100
+            text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}]  {text}'
+
+        sys.stderr.write(f'\r{text[:width]}\033[K')
+        sys.stderr.flush()
+        self._shown = True
+
+        return True
+
+    def clear(self) -> None:
+        """Remove the bar, so that log messages start on a clean line"""
+
+        if self._shown:
+            sys.stderr.write('\r\033[K')
+            sys.stderr.flush()
+
+
 class Backup:
     """Main class defining parameters and functions for performing backup
 
@@ -124,6 +181,8 @@ class Backup:
             Run remote rsync with sudo
         remove_before: bool
             Indicate if removing old backups will be performed before copying files
+        progress: bool
+            Show a progress bar while rsync runs
 
     Methods:
         check_params():
@@ -141,7 +200,8 @@ class Backup:
     def __init__(self, inputs: Optional[List[str]], output: Optional[str], exclude: Optional[List[str]],
                  keep: Optional[int], options: str,
                  ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None,
-                 remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False) -> None:
+                 remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False,
+                 progress: bool = False) -> None:
         # Missing values are reported by check_params()
         self.inputs = inputs if inputs is not None else []
         self.output = output if output is not None else ''
@@ -154,6 +214,7 @@ class Backup:
         self.remote_sudo = remote_sudo
         self._remove_before = remove_before
         self._verbose = verbose
+        self._progress = progress
         self._last_backup = ''
         self._server = ''
         self._output_dir = ''
@@ -648,11 +709,25 @@ class Backup:
         if self._remote and self.remote_sudo:
             args.append('--rsync-path=sudo rsync')
 
+        bar = None
+
+        if self._progress:
+            args.append('--info=progress2')
+            bar = _ProgressBar()
+
         with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p:
             assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
             p.stdin.close()
 
-            output = [r for r in _read_records(p.stdout) if r != '']
+            output = []
+
+            try:
+                for r in _read_records(p.stdout):
+                    if r != '' and (bar is None or not bar.update(r)):
+                        output.append(r)
+            finally:
+                if bar is not None:
+                    bar.clear()
 
             try:
                 del os.environ['SSHPASS']
@@ -766,6 +841,8 @@ def _parse_arguments() -> argparse.Namespace:
                         help='Remove old backups before executing the backup, instead of after')
     parser.add_argument('--no-syslog', action='store_true',
                         help='Disable systemd journal logging')
+    parser.add_argument('--no-progress', action='store_true',
+                        help='Don\'t show a progress bar (it is only shown on a terminal)')
     parser.add_argument('--rsync-options', nargs='+',
                         choices=['a', 'l', 'p', 't', 'g', 'o',
                                  'c', 'h', 's', 'D', 'H', 'X'],
@@ -1039,7 +1116,8 @@ def simple_backup() -> int:
         rsync_options.append('--numeric-ids')
 
     backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile,
-                    remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose)
+                    remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose,
+                    progress=not args.no_progress and sys.stderr.isatty())
 
     try:
         return_code = backup.check_params(homedir)

tests/test_cli.py

@@ -27,7 +27,7 @@ class FakeBackup:
     def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str,
                  ssh_host: Optional[str] = None, ssh_user: Optional[str] = None,
                  ssh_keyfile: Optional[str] = None, remote_sudo: bool = False,
-                 remove_before: bool = False, verbose: bool = False) -> None:
+                 remove_before: bool = False, verbose: bool = False, progress: bool = False) -> None:
         self.inputs = inputs
         self.output = output
         self.exclude = exclude
@@ -39,6 +39,7 @@ class FakeBackup:
         self.remote_sudo = remote_sudo
         self.remove_before = remove_before
         self.verbose = verbose
+        self.progress = progress
         self.homedir: Optional[str] = None
         self.ran = False
         self.closed = False
@@ -224,6 +225,18 @@ ssh_keyfile=/keys/id
         b = last_backup()
         assert b.remote_sudo and b.remove_before and b.verbose
 
+    @pytest.mark.parametrize('tty, argv, progress', [
+        (True, [], True),
+        (True, ['--no-progress'], False),
+        (False, [], False),
+    ])
+    def test_progress(self, run_cli: RunCli, config: Callable[[str], str], monkeypatch: pytest.MonkeyPatch,
+                      tty: bool, argv: list[str], progress: bool) -> None:
+        monkeypatch.setattr(sb.sys.stderr, 'isatty', lambda: tty)
+
+        assert run_cli('-c', config('backup_dir=/backups'), *argv) == 0
+        assert last_backup().progress is progress
+
     def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None:
         FakeBackup.check_code = 2
 

tests/test_local.py

@@ -53,8 +53,9 @@ def out(tmp_path: Path) -> Path:
 
 
 def make_backup(out: Path, inputs: list[str], keep: int = -1, exclude: Optional[list[str]] = None,
-                remove_before: bool = False) -> sb.Backup:
+                remove_before: bool = False, progress: bool = False) -> sb.Backup:
-    backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before)
+    backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before,
+                       progress=progress)
 
     assert backup.check_params() == 0
 
@@ -187,6 +188,16 @@ class TestRsync:
         assert run_backup(out, [str(src)], keep=0) == 0
         assert backups(out) == ['2026-10-14_12-00-00']
 
+    def test_progress(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture,
+                      capsys: pytest.CaptureFixture[str]) -> None:
+        assert run_backup(out, [str(src)], progress=True) == 0
+
+        assert '100%' in capsys.readouterr().err
+        # Only the summary is logged, not the progress updates
+        assert 'rsync: sent' in caplog.text
+        assert 'rsync: total size' in caplog.text
+        assert '%' not in caplog.text
+
     def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None:
         config_file = str(tmp_path / 'missing.conf')
 
@@ -219,6 +230,43 @@ class TestReadRecords:
         assert list(sb._read_records(stream)) == ['file', '', 'namé', '�']
 
 
+class TestProgressBar:
+    @pytest.fixture(autouse=True)
+    def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24)))
+        # 30 s elapsed at each update
+        monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
+
+    def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
+        bar = sb._ProgressBar()
+
+        assert bar.update('          2.00M  25%    1.47MB/s    0:00:02 (xfr#1, ir-chk=2/4)')
+        assert capsys.readouterr().err == \
+            '\r[#######-----------------------]   25%  2.00M  1.47MB/s  ETA 0:01:30  (scanning)\033[K'
+
+    def test_without_human_readable(self, capsys: pytest.CaptureFixture[str]) -> None:
+        bar = sb._ProgressBar()
+
+        assert bar.update('      4,000,000  50%    2.97MB/s    0:00:01 (xfr#2, to-chk=1/4)')
+        assert capsys.readouterr().err.endswith(']   50%  4,000,000  2.97MB/s  ETA 0:00:30\033[K')
+
+    def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None:
+        bar = sb._ProgressBar()
+
+        for record in ['home/user/file 50% 1:00:00', 'sent 1 bytes  received 2 bytes', 'total size is 3']:
+            assert not bar.update(record)
+
+        bar.clear()
+        assert capsys.readouterr().err == ''
+
+    def test_narrow_terminal(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
+        monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((20, 24)))
+
+        assert sb._ProgressBar().update('  6.00M 100%    1.48MB/s    0:00:03 (xfr#3, to-chk=0/4)')
+        # No room for the bar
+        assert capsys.readouterr().err == '\r100%  6.00M  1.48MB\033[K'
+
+
 class FakePopen:
     """Stands in for rsync: creates the destination and returns a chosen return code"""