From 7cd90f8d110b3b2b8dc0925062c445ba3e6928e7 Mon Sep 17 00:00:00 2001 From: Fuxino Date: Wed, 14 Oct 2026 09:46:50 +0200 Subject: [PATCH] Add a progress bar for rsync On a terminal, run rsync with --info=progress2 and draw its updates as a progress bar on stderr. The ETA is estimated locally, since rsync reports the elapsed time when a file finishes. Disable it with --no-progress. --- man/simple_backup.1 | 6 +++ simple_backup/simple_backup.py | 86 ++++++++++++++++++++++++++++++++-- tests/test_cli.py | 15 +++++- tests/test_local.py | 52 +++++++++++++++++++- 4 files changed, 152 insertions(+), 7 deletions(-) diff --git a/man/simple_backup.1 b/man/simple_backup.1 index 68c80e2..eef8486 100644 --- a/man/simple_backup.1 +++ b/man/simple_backup.1 @@ -105,6 +105,12 @@ according to \-\-keep will be left, and the new backup may be incomplete. In par .B \-\-no\-syslog Don't use systemd journal for logging. .TP +.B \-\-no\-progress +Don't show a progress bar while rsync copies the files. The progress bar is only shown when +the output is a terminal, so it never appears in the systemd journal or in redirected output. +While rsync is still looking for files, the bar shows \(aq(scanning)\(aq, and the percentage +and estimated time are only approximate. +.TP .B \-\-rsync\-options OPTIONS [OPTION...] By default, the following rsync options are used: .RS diff --git a/simple_backup/simple_backup.py b/simple_backup/simple_backup.py index e73fb98..34dfa88 100755 --- a/simple_backup/simple_backup.py +++ b/simple_backup/simple_backup.py @@ -23,7 +23,7 @@ import re from typing import Callable, Iterator, List, Optional, ParamSpec, TypedDict, TypeVar import warnings from functools import wraps -from shutil import rmtree, which +from shutil import get_terminal_size, rmtree, which import shlex import argparse import configparser @@ -74,6 +74,10 @@ R = TypeVar('R') # Seconds to wait for the TCP connection to the SSH server SSH_TIMEOUT = 15 +# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N) +PROGRESS_RE = re.compile(r'\s*(\S+)\s+(\d+)%\s+(\S+)\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=\d+/\d+\))?\s*') +PROGRESS_BAR_WIDTH = 30 + def timing(func: Callable[P, R]) -> Callable[P, R]: """Decorator to measure execution time of a function @@ -100,6 +104,59 @@ class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFo """Custom format for argparse help text""" +class _ProgressBar: + """Draws rsync --info=progress2 updates as a bar on stderr""" + + def __init__(self) -> None: + self._start = default_timer() + # rsync reports ir-chk while incremental recursion is still finding files + self._scanning = True + self._shown = False + + def update(self, record: str) -> bool: + """Draw the bar if record is a progress update, and return whether it was""" + + m = PROGRESS_RE.fullmatch(record) + + if m is None: + return False + + size, percent, speed, chk = m.group(1), int(m.group(2)), m.group(3), m.group(4) + + if chk is not None: + self._scanning = chk == 'ir' + + # rsync's own time field is the elapsed time on records ending a file, so estimate the ETA here + if 0 < percent < 100: + eta = int((default_timer() - self._start) * (100 - percent) / percent) + text = f'{percent:3d}% {size} {speed} ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}' + else: + text = f'{percent:3d}% {size} {speed}' + + if self._scanning: + text += ' (scanning)' + + width = get_terminal_size().columns - 1 + + # Fixed width, so the bar doesn't jump when the text changes length + if len(text) + PROGRESS_BAR_WIDTH + 4 <= width: + filled = PROGRESS_BAR_WIDTH * percent // 100 + text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}] {text}' + + sys.stderr.write(f'\r{text[:width]}\033[K') + sys.stderr.flush() + self._shown = True + + return True + + def clear(self) -> None: + """Remove the bar, so that log messages start on a clean line""" + + if self._shown: + sys.stderr.write('\r\033[K') + sys.stderr.flush() + + class Backup: """Main class defining parameters and functions for performing backup @@ -124,6 +181,8 @@ class Backup: Run remote rsync with sudo remove_before: bool Indicate if removing old backups will be performed before copying files + progress: bool + Show a progress bar while rsync runs Methods: check_params(): @@ -141,7 +200,8 @@ class Backup: def __init__(self, inputs: Optional[List[str]], output: Optional[str], exclude: Optional[List[str]], keep: Optional[int], options: str, ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None, - remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False) -> None: + remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False, + progress: bool = False) -> None: # Missing values are reported by check_params() self.inputs = inputs if inputs is not None else [] self.output = output if output is not None else '' @@ -154,6 +214,7 @@ class Backup: self.remote_sudo = remote_sudo self._remove_before = remove_before self._verbose = verbose + self._progress = progress self._last_backup = '' self._server = '' self._output_dir = '' @@ -648,11 +709,25 @@ class Backup: if self._remote and self.remote_sudo: args.append('--rsync-path=sudo rsync') + bar = None + + if self._progress: + args.append('--info=progress2') + bar = _ProgressBar() + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p: assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase) p.stdin.close() - output = [r for r in _read_records(p.stdout) if r != ''] + output = [] + + try: + for r in _read_records(p.stdout): + if r != '' and (bar is None or not bar.update(r)): + output.append(r) + finally: + if bar is not None: + bar.clear() try: del os.environ['SSHPASS'] @@ -766,6 +841,8 @@ def _parse_arguments() -> argparse.Namespace: help='Remove old backups before executing the backup, instead of after') parser.add_argument('--no-syslog', action='store_true', help='Disable systemd journal logging') + parser.add_argument('--no-progress', action='store_true', + help='Don\'t show a progress bar (it is only shown on a terminal)') parser.add_argument('--rsync-options', nargs='+', choices=['a', 'l', 'p', 't', 'g', 'o', 'c', 'h', 's', 'D', 'H', 'X'], @@ -1039,7 +1116,8 @@ def simple_backup() -> int: rsync_options.append('--numeric-ids') backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile, - remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose) + remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose, + progress=not args.no_progress and sys.stderr.isatty()) try: return_code = backup.check_params(homedir) diff --git a/tests/test_cli.py b/tests/test_cli.py index 57254e6..30fe202 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -27,7 +27,7 @@ class FakeBackup: def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str, ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None, remote_sudo: bool = False, - remove_before: bool = False, verbose: bool = False) -> None: + remove_before: bool = False, verbose: bool = False, progress: bool = False) -> None: self.inputs = inputs self.output = output self.exclude = exclude @@ -39,6 +39,7 @@ class FakeBackup: self.remote_sudo = remote_sudo self.remove_before = remove_before self.verbose = verbose + self.progress = progress self.homedir: Optional[str] = None self.ran = False self.closed = False @@ -224,6 +225,18 @@ ssh_keyfile=/keys/id b = last_backup() assert b.remote_sudo and b.remove_before and b.verbose + @pytest.mark.parametrize('tty, argv, progress', [ + (True, [], True), + (True, ['--no-progress'], False), + (False, [], False), + ]) + def test_progress(self, run_cli: RunCli, config: Callable[[str], str], monkeypatch: pytest.MonkeyPatch, + tty: bool, argv: list[str], progress: bool) -> None: + monkeypatch.setattr(sb.sys.stderr, 'isatty', lambda: tty) + + assert run_cli('-c', config('backup_dir=/backups'), *argv) == 0 + assert last_backup().progress is progress + def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None: FakeBackup.check_code = 2 diff --git a/tests/test_local.py b/tests/test_local.py index 0f2da3c..9fb1363 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -53,8 +53,9 @@ def out(tmp_path: Path) -> Path: def make_backup(out: Path, inputs: list[str], keep: int = -1, exclude: Optional[list[str]] = None, - remove_before: bool = False) -> sb.Backup: - backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before) + remove_before: bool = False, progress: bool = False) -> sb.Backup: + backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before, + progress=progress) assert backup.check_params() == 0 @@ -187,6 +188,16 @@ class TestRsync: assert run_backup(out, [str(src)], keep=0) == 0 assert backups(out) == ['2026-10-14_12-00-00'] + def test_progress(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture, + capsys: pytest.CaptureFixture[str]) -> None: + assert run_backup(out, [str(src)], progress=True) == 0 + + assert '100%' in capsys.readouterr().err + # Only the summary is logged, not the progress updates + assert 'rsync: sent' in caplog.text + assert 'rsync: total size' in caplog.text + assert '%' not in caplog.text + def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None: config_file = str(tmp_path / 'missing.conf') @@ -219,6 +230,43 @@ class TestReadRecords: assert list(sb._read_records(stream)) == ['file', '', 'namé', '�'] +class TestProgressBar: + @pytest.fixture(autouse=True) + def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24))) + # 30 s elapsed at each update + monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__) + + def test_update(self, capsys: pytest.CaptureFixture[str]) -> None: + bar = sb._ProgressBar() + + assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=2/4)') + assert capsys.readouterr().err == \ + '\r[#######-----------------------] 25% 2.00M 1.47MB/s ETA 0:01:30 (scanning)\033[K' + + def test_without_human_readable(self, capsys: pytest.CaptureFixture[str]) -> None: + bar = sb._ProgressBar() + + assert bar.update(' 4,000,000 50% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)') + assert capsys.readouterr().err.endswith('] 50% 4,000,000 2.97MB/s ETA 0:00:30\033[K') + + def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None: + bar = sb._ProgressBar() + + for record in ['home/user/file 50% 1:00:00', 'sent 1 bytes received 2 bytes', 'total size is 3']: + assert not bar.update(record) + + bar.clear() + assert capsys.readouterr().err == '' + + def test_narrow_terminal(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((20, 24))) + + assert sb._ProgressBar().update(' 6.00M 100% 1.48MB/s 0:00:03 (xfr#3, to-chk=0/4)') + # No room for the bar + assert capsys.readouterr().err == '\r100% 6.00M 1.48MB\033[K' + + class FakePopen: """Stands in for rsync: creates the destination and returns a chosen return code"""