From 8aef768a23cd9b66d9a3019681f03cea2a66c896 Mon Sep 17 00:00:00 2001 From: Fuxino Date: Sat, 21 Nov 2026 17:04:49 +0100 Subject: [PATCH] Base the progress ETA on the last 30 seconds and show the copy speed The whole-run file rate is dominated by hard-linked files, so after a big folder moved the ETA showed seconds while gigabytes were still to be copied. --- README.md | 3 +- man/simple_backup.1 | 3 ++ simple_backup/simple_backup.py | 79 ++++++++++++++++++++++++++++++++-- tests/test_local.py | 62 +++++++++++++++++++------- 4 files changed, 127 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 32739de..30aa5cb 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,8 @@ In that case the user who ran the command is detected automatically, so that '~' When running as root in other ways (e.g. from a systemd service), specify the user with the --user (or -u) argument. ## Progress -When run in a terminal, simple_backup shows a progress bar with the percentage of files copied and an estimated time to completion. +When run in a terminal, simple_backup shows a progress bar with the percentage of files copied, the copy speed and an estimated time to completion. +Speed and estimated time follow the last 30 seconds, so the estimate adapts when big files are being copied. Disable it with --no-progress. ## Old backups diff --git a/man/simple_backup.1 b/man/simple_backup.1 index 51a7ff9..067ce5f 100644 --- a/man/simple_backup.1 +++ b/man/simple_backup.1 @@ -116,6 +116,9 @@ Don't use systemd journal for logging. .B \-\-no\-progress Don't show a progress bar while rsync copies the files. The progress bar is only shown when the output is a terminal, so it never appears in the systemd journal or in redirected output. +It shows the percentage of files copied, the copy speed and the estimated time, both from the +last 30 seconds; the time shows \(aq\-\-:\-\-:\-\-\(aq when no file finished in that time (e.g. while +copying a big file). While rsync is still looking for files, the percentage and estimated time (marked with \(aq~\(aq) are based on the number of files in the last backup (or on the files found so far, once this backup has more), and stay below 100%. Without a previous count the bar shows diff --git a/simple_backup/simple_backup.py b/simple_backup/simple_backup.py index 7319510..59b283d 100755 --- a/simple_backup/simple_backup.py +++ b/simple_backup/simple_backup.py @@ -74,6 +74,10 @@ SSH_TIMEOUT = 15 # An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N) PROGRESS_RE = re.compile(r'\s*(\S+)\s+\d+%\s+\S+\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=(\d+)/(\d+)\))?\s*') PROGRESS_BAR_WIDTH = 30 +# rsync's transferred size: 1,234B, 1.23MB (-h), 1.17MiB (-hh) or 1.23M (older rsync) +SIZE_RE = re.compile(r'([\d.,]+)([KMGTP]?)(i?)B?') +# Seconds of recent progress the ETA and speed come from, so they follow changes of pace +RATE_WINDOW = 30 def _format_duration(seconds: float) -> str: @@ -87,6 +91,43 @@ def _format_duration(seconds: float) -> str: return f'{s // 60:02d}:{s % 60:02d}' +def _parse_size(size: str) -> int | None: + """Bytes in an rsync size field, or None if it can't be parsed""" + + m = SIZE_RE.fullmatch(size) + + if m is None: + return None + + number, unit, binary = m.groups() + + try: + if unit == '': + return int(number.replace(',', '').replace('.', '')) + + # Suffixed sizes use the locale's decimal point + return int(float(number.replace(',', '.')) * (1024 if binary else 1000) ** ('KMGTP'.index(unit) + 1)) + except ValueError: + return None + + +def _format_speed(speed: float) -> str: + """Format bytes per second like rsync -h""" + + if speed < 1000: + return f'{speed:.0f}B/s' + + units = ['KB', 'MB', 'GB', 'TB'] + i = 0 + speed /= 1000 + + while speed >= 1000 and i < len(units) - 1: + speed /= 1000 + i += 1 + + return f'{speed:.1f}{units[i]}/s' + + class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter): """Custom format for argparse help text""" @@ -103,6 +144,9 @@ class _ProgressBar: # Total files of the last backup, used until rsync knows the real total self._estimate = estimate self._shown = False + # (time, files checked, bytes copied) about once a second, back to the start of the rate window + self._samples: deque[tuple[float, int, int]] = deque([(self._start, 0, 0)]) + self._file_rate: float | None = None @property def total(self) -> int | None: @@ -110,8 +154,33 @@ class _ProgressBar: return None if self._scanning or self._total == 0 else self._total + def _sample(self, copied: int | None) -> str: + """Update the file rate over the window, and return the copy speed text (empty if unknown)""" + + now = default_timer() + + if now - self._samples[-1][0] >= 1: + self._samples.append((now, self._checked, copied or 0)) + + # Keep the newest sample at or before the window start as the base + while len(self._samples) > 1 and self._samples[1][0] <= now - RATE_WINDOW: + self._samples.popleft() + + t0, checked0, copied0 = self._samples[0] + span = now - t0 + self._file_rate = (self._checked - checked0) / span if span > 0 else None + + if copied is None or span <= 0: + return '' + + return f' {_format_speed(max(copied - copied0, 0) / span)}' + def _eta(self, total: int) -> str: - eta = int((default_timer() - self._start) * (total - self._checked) / self._checked) + # No file finished within the window (e.g. while copying a big one): no basis for an estimate + if not self._file_rate: + return '--:--:--' + + eta = int((total - self._checked) / self._file_rate) return f'{eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}' @@ -131,6 +200,8 @@ class _ProgressBar: self._total = int(m.group(4)) self._checked = self._total - int(m.group(3)) + speed = self._sample(_parse_size(size)) + # Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out percent: int | None @@ -141,16 +212,16 @@ class _ProgressBar: if self._estimate > 0 and self._checked > 0: # The real total is still unknown, so never claim 100% percent = min(self._checked * 100 // estimate, 99) - text = f'{f"~{percent}":>3}% {self._checked:,}/~{estimate:,} files {size} copied' + text = f'{f"~{percent}":>3}% {self._checked:,}/~{estimate:,} files {size} copied{speed}' if percent > 0: text += f' ETA ~{self._eta(estimate)}' else: - text = f'{self._checked:,} files checked {size} copied (scanning)' + text = f'{self._checked:,} files checked {size} copied{speed} (scanning)' percent = None else: percent = self._checked * 100 // self._total - text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied' + text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied{speed}' if 0 < percent < 100: text += f' ETA {self._eta(self._total)}' diff --git a/tests/test_local.py b/tests/test_local.py index eb1a465..d752d4d 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -263,10 +263,26 @@ def test_format_duration(seconds: float, text: str) -> None: assert sb._format_duration(seconds) == text +@pytest.mark.parametrize(('size', 'size_bytes'), [ + ('30,000,000B', 30_000_000), ('4,000,000', 4_000_000), ('30.0MB', 30_000_000), ('30,0MB', 30_000_000), + ('2.00M', 2_000_000), ('1.5GB', 1_500_000_000), ('28.6MiB', 29_989_273), ('512B', 512), ('?', None), + ('1.2.3MB', None), +]) +def test_parse_size(size: str, size_bytes: int | None) -> None: + assert sb._parse_size(size) == size_bytes + + +@pytest.mark.parametrize(('speed', 'text'), [ + (0, '0B/s'), (999, '999B/s'), (1500, '1.5KB/s'), (10_500_000, '10.5MB/s'), (2e9, '2.0GB/s'), (3e12, '3.0TB/s'), +]) +def test_format_speed(speed: float, text: str) -> None: + assert sb._format_speed(speed) == text + + class TestProgressBar: @pytest.fixture(autouse=True) def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24))) + monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24))) # 30 s elapsed at each update monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__) @@ -274,7 +290,7 @@ class TestProgressBar: bar = sb._ProgressBar() assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1009/2632)') - assert capsys.readouterr().err == '\r1,623 files checked 2.00M copied (scanning)\033[K' + assert capsys.readouterr().err == '\r1,623 files checked 2.00M copied 66.7KB/s (scanning)\033[K' def test_update(self, capsys: pytest.CaptureFixture[str]) -> None: bar = sb._ProgressBar() @@ -282,7 +298,7 @@ class TestProgressBar: # With --link-dest rsync's percentage leaves out linked files, so the files are counted instead assert bar.update(' 4,000,000 1% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)') assert capsys.readouterr().err == \ - '\r[######################--------] 75% 3/4 files 4,000,000 copied ETA 0:00:10\033[K' + '\r[######################--------] 75% 3/4 files 4,000,000 copied 133.3KB/s ETA 0:00:10\033[K' def test_keeps_counts_within_file(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: @@ -291,43 +307,59 @@ class TestProgressBar: assert bar.update(' 2.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=2/4)') assert bar.update(' 3.00M 2% 1.47MB/s 0:00:03') - assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied ETA 0:00:30\033[K') + assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied 100.0KB/s ETA 0:00:30\033[K') - def test_estimate_while_scanning(self, monkeypatch: pytest.MonkeyPatch, + def test_eta_follows_recent_pace(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24))) + monkeypatch.setattr(sb, 'default_timer', iter([0.0, 100.0, 130.0]).__next__) + bar = sb._ProgressBar() + + # Fast while linking unchanged files, then one file in the last 30 s + assert bar.update(' 10.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=100/1000)') + assert capsys.readouterr().err.endswith(' 100.0KB/s ETA 0:00:11\033[K') + assert bar.update(' 70.00M 7% 1.47MB/s 0:00:02 (xfr#2, to-chk=99/1000)') + assert capsys.readouterr().err.endswith(' 2.0MB/s ETA 0:49:30\033[K') + + def test_eta_unknown_within_big_file(self, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sb, 'default_timer', iter([0.0, 10.0, 50.0]).__next__) + bar = sb._ProgressBar() + + assert bar.update(' 10.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=10/20)') + # No file finished in the last 30 s + assert bar.update(' 50.00M 5% 1.47MB/s 0:00:02') + assert capsys.readouterr().err.endswith(' 50.00M copied 1.0MB/s ETA --:--:--\033[K') + + def test_estimate_while_scanning(self, capsys: pytest.CaptureFixture[str]) -> None: bar = sb._ProgressBar(estimate=4000) assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1000/2000)') assert bar.total is None assert capsys.readouterr().err == \ - '\r[#######-----------------------] ~25% 1,000/~4,000 files 2.00M copied ETA ~0:01:30\033[K' + '\r[#######-----------------------] ~25% 1,000/~4,000 files 2.00M copied 66.7KB/s ETA ~0:01:30\033[K' - def test_estimate_exceeded(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24))) + def test_estimate_exceeded(self, capsys: pytest.CaptureFixture[str]) -> None: bar = sb._ProgressBar(estimate=1000) # More files than in the last backup: estimate from the files found so far assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=500/2000)') assert capsys.readouterr().err == \ - '\r[######################--------] ~75% 1,500/~2,000 files 2.00M copied ETA ~0:00:10\033[K' + '\r[######################--------] ~75% 1,500/~2,000 files 2.00M copied 66.7KB/s ETA ~0:00:10\033[K' - def test_estimate_capped_while_scanning(self, monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str]) -> None: - monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24))) + def test_estimate_capped_while_scanning(self, capsys: pytest.CaptureFixture[str]) -> None: bar = sb._ProgressBar(estimate=1000) # All files found so far are checked, but rsync may still find more assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)') assert bar.total is None - assert ' ~99% 2,000/~2,000 files 2.00M copied ETA ~0:00:00\033[K' in capsys.readouterr().err + assert ' ~99% 2,000/~2,000 files 2.00M copied 66.7KB/s ETA ~0:00:00\033[K' in capsys.readouterr().err def test_no_estimate_without_last_count(self, capsys: pytest.CaptureFixture[str]) -> None: bar = sb._ProgressBar() # The files found so far are only a guess at the total near the end of the scan assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)') - assert capsys.readouterr().err == '\r2,000 files checked 2.00M copied (scanning)\033[K' + assert capsys.readouterr().err == '\r2,000 files checked 2.00M copied 66.7KB/s (scanning)\033[K' def test_real_total_replaces_estimate(self, capsys: pytest.CaptureFixture[str]) -> None: bar = sb._ProgressBar(estimate=4000)