Base the progress ETA on the last 30 seconds and show the copy speed

The whole-run file rate is dominated by hard-linked files, so after a big
folder moved the ETA showed seconds while gigabytes were still to be copied.
This commit is contained in:
Fuxino committed 2026-11-21 17:04:49 +01:00
1 parent efdff589f7
commit 8aef768a23
4 files changed
+127 -20

No files matched your search

+2 -1
View File
@@ -59,7 +59,8 @@ In that case the user who ran the command is detected automatically, so that '~'
When running as root in other ways (e.g. from a systemd service), specify the user with the --user (or -u) argument. When running as root in other ways (e.g. from a systemd service), specify the user with the --user (or -u) argument.
## Progress ## Progress
When run in a terminal, simple_backup shows a progress bar with the percentage of files copied and an estimated time to completion. When run in a terminal, simple_backup shows a progress bar with the percentage of files copied, the copy speed and an estimated time to completion.
Speed and estimated time follow the last 30 seconds, so the estimate adapts when big files are being copied.
Disable it with --no-progress. Disable it with --no-progress.
## Old backups ## Old backups
+3
View File
@@ -116,6 +116,9 @@ Don't use systemd journal for logging.
.B \-\-no\-progress .B \-\-no\-progress
Don't show a progress bar while rsync copies the files. The progress bar is only shown when Don't show a progress bar while rsync copies the files. The progress bar is only shown when
the output is a terminal, so it never appears in the systemd journal or in redirected output. the output is a terminal, so it never appears in the systemd journal or in redirected output.
It shows the percentage of files copied, the copy speed and the estimated time, both from the
last 30 seconds; the time shows \(aq\-\-:\-\-:\-\-\(aq when no file finished in that time (e.g. while
copying a big file).
While rsync is still looking for files, the percentage and estimated time (marked with While rsync is still looking for files, the percentage and estimated time (marked with
\(aq~\(aq) are based on the number of files in the last backup (or on the files found so far, \(aq~\(aq) are based on the number of files in the last backup (or on the files found so far,
once this backup has more), and stay below 100%. Without a previous count the bar shows once this backup has more), and stay below 100%. Without a previous count the bar shows
+75 -4
View File
@@ -74,6 +74,10 @@ SSH_TIMEOUT = 15
# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N) # An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
PROGRESS_RE = re.compile(r'\s*(\S+)\s+\d+%\s+\S+\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=(\d+)/(\d+)\))?\s*') PROGRESS_RE = re.compile(r'\s*(\S+)\s+\d+%\s+\S+\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=(\d+)/(\d+)\))?\s*')
PROGRESS_BAR_WIDTH = 30 PROGRESS_BAR_WIDTH = 30
# rsync's transferred size: 1,234B, 1.23MB (-h), 1.17MiB (-hh) or 1.23M (older rsync)
SIZE_RE = re.compile(r'([\d.,]+)([KMGTP]?)(i?)B?')
# Seconds of recent progress the ETA and speed come from, so they follow changes of pace
RATE_WINDOW = 30
def _format_duration(seconds: float) -> str: def _format_duration(seconds: float) -> str:
@@ -87,6 +91,43 @@ def _format_duration(seconds: float) -> str:
return f'{s // 60:02d}:{s % 60:02d}' return f'{s // 60:02d}:{s % 60:02d}'
def _parse_size(size: str) -> int | None:
"""Bytes in an rsync size field, or None if it can't be parsed"""
m = SIZE_RE.fullmatch(size)
if m is None:
return None
number, unit, binary = m.groups()
try:
if unit == '':
return int(number.replace(',', '').replace('.', ''))
# Suffixed sizes use the locale's decimal point
return int(float(number.replace(',', '.')) * (1024 if binary else 1000) ** ('KMGTP'.index(unit) + 1))
except ValueError:
return None
def _format_speed(speed: float) -> str:
"""Format bytes per second like rsync -h"""
if speed < 1000:
return f'{speed:.0f}B/s'
units = ['KB', 'MB', 'GB', 'TB']
i = 0
speed /= 1000
while speed >= 1000 and i < len(units) - 1:
speed /= 1000
i += 1
return f'{speed:.1f}{units[i]}/s'
class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter): class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
"""Custom format for argparse help text""" """Custom format for argparse help text"""
@@ -103,6 +144,9 @@ class _ProgressBar:
# Total files of the last backup, used until rsync knows the real total # Total files of the last backup, used until rsync knows the real total
self._estimate = estimate self._estimate = estimate
self._shown = False self._shown = False
# (time, files checked, bytes copied) about once a second, back to the start of the rate window
self._samples: deque[tuple[float, int, int]] = deque([(self._start, 0, 0)])
self._file_rate: float | None = None
@property @property
def total(self) -> int | None: def total(self) -> int | None:
@@ -110,8 +154,33 @@ class _ProgressBar:
return None if self._scanning or self._total == 0 else self._total return None if self._scanning or self._total == 0 else self._total
def _sample(self, copied: int | None) -> str:
"""Update the file rate over the window, and return the copy speed text (empty if unknown)"""
now = default_timer()
if now - self._samples[-1][0] >= 1:
self._samples.append((now, self._checked, copied or 0))
# Keep the newest sample at or before the window start as the base
while len(self._samples) > 1 and self._samples[1][0] <= now - RATE_WINDOW:
self._samples.popleft()
t0, checked0, copied0 = self._samples[0]
span = now - t0
self._file_rate = (self._checked - checked0) / span if span > 0 else None
if copied is None or span <= 0:
return ''
return f' {_format_speed(max(copied - copied0, 0) / span)}'
def _eta(self, total: int) -> str: def _eta(self, total: int) -> str:
eta = int((default_timer() - self._start) * (total - self._checked) / self._checked) # No file finished within the window (e.g. while copying a big one): no basis for an estimate
if not self._file_rate:
return '--:--:--'
eta = int((total - self._checked) / self._file_rate)
return f'{eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}' return f'{eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
@@ -131,6 +200,8 @@ class _ProgressBar:
self._total = int(m.group(4)) self._total = int(m.group(4))
self._checked = self._total - int(m.group(3)) self._checked = self._total - int(m.group(3))
speed = self._sample(_parse_size(size))
# Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out # Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out
percent: int | None percent: int | None
@@ -141,16 +212,16 @@ class _ProgressBar:
if self._estimate > 0 and self._checked > 0: if self._estimate > 0 and self._checked > 0:
# The real total is still unknown, so never claim 100% # The real total is still unknown, so never claim 100%
percent = min(self._checked * 100 // estimate, 99) percent = min(self._checked * 100 // estimate, 99)
text = f'{f"~{percent}":>3}% {self._checked:,}/~{estimate:,} files {size} copied' text = f'{f"~{percent}":>3}% {self._checked:,}/~{estimate:,} files {size} copied{speed}'
if percent > 0: if percent > 0:
text += f' ETA ~{self._eta(estimate)}' text += f' ETA ~{self._eta(estimate)}'
else: else:
text = f'{self._checked:,} files checked {size} copied (scanning)' text = f'{self._checked:,} files checked {size} copied{speed} (scanning)'
percent = None percent = None
else: else:
percent = self._checked * 100 // self._total percent = self._checked * 100 // self._total
text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied' text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied{speed}'
if 0 < percent < 100: if 0 < percent < 100:
text += f' ETA {self._eta(self._total)}' text += f' ETA {self._eta(self._total)}'
+47 -15
View File
@@ -263,10 +263,26 @@ def test_format_duration(seconds: float, text: str) -> None:
assert sb._format_duration(seconds) == text assert sb._format_duration(seconds) == text
@pytest.mark.parametrize(('size', 'size_bytes'), [
('30,000,000B', 30_000_000), ('4,000,000', 4_000_000), ('30.0MB', 30_000_000), ('30,0MB', 30_000_000),
('2.00M', 2_000_000), ('1.5GB', 1_500_000_000), ('28.6MiB', 29_989_273), ('512B', 512), ('?', None),
('1.2.3MB', None),
])
def test_parse_size(size: str, size_bytes: int | None) -> None:
assert sb._parse_size(size) == size_bytes
@pytest.mark.parametrize(('speed', 'text'), [
(0, '0B/s'), (999, '999B/s'), (1500, '1.5KB/s'), (10_500_000, '10.5MB/s'), (2e9, '2.0GB/s'), (3e12, '3.0TB/s'),
])
def test_format_speed(speed: float, text: str) -> None:
assert sb._format_speed(speed) == text
class TestProgressBar: class TestProgressBar:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None: def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24))) monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
# 30 s elapsed at each update # 30 s elapsed at each update
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__) monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
@@ -274,7 +290,7 @@ class TestProgressBar:
bar = sb._ProgressBar() bar = sb._ProgressBar()
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1009/2632)') assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1009/2632)')
assert capsys.readouterr().err == '\r1,623 files checked 2.00M copied (scanning)\033[K' assert capsys.readouterr().err == '\r1,623 files checked 2.00M copied 66.7KB/s (scanning)\033[K'
def test_update(self, capsys: pytest.CaptureFixture[str]) -> None: def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar() bar = sb._ProgressBar()
@@ -282,7 +298,7 @@ class TestProgressBar:
# With --link-dest rsync's percentage leaves out linked files, so the files are counted instead # With --link-dest rsync's percentage leaves out linked files, so the files are counted instead
assert bar.update(' 4,000,000 1% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)') assert bar.update(' 4,000,000 1% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)')
assert capsys.readouterr().err == \ assert capsys.readouterr().err == \
'\r[######################--------] 75% 3/4 files 4,000,000 copied ETA 0:00:10\033[K' '\r[######################--------] 75% 3/4 files 4,000,000 copied 133.3KB/s ETA 0:00:10\033[K'
def test_keeps_counts_within_file(self, monkeypatch: pytest.MonkeyPatch, def test_keeps_counts_within_file(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None: capsys: pytest.CaptureFixture[str]) -> None:
@@ -291,43 +307,59 @@ class TestProgressBar:
assert bar.update(' 2.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=2/4)') assert bar.update(' 2.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=2/4)')
assert bar.update(' 3.00M 2% 1.47MB/s 0:00:03') assert bar.update(' 3.00M 2% 1.47MB/s 0:00:03')
assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied ETA 0:00:30\033[K') assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied 100.0KB/s ETA 0:00:30\033[K')
def test_estimate_while_scanning(self, monkeypatch: pytest.MonkeyPatch, def test_eta_follows_recent_pace(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None: capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24))) monkeypatch.setattr(sb, 'default_timer', iter([0.0, 100.0, 130.0]).__next__)
bar = sb._ProgressBar()
# Fast while linking unchanged files, then one file in the last 30 s
assert bar.update(' 10.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=100/1000)')
assert capsys.readouterr().err.endswith(' 100.0KB/s ETA 0:00:11\033[K')
assert bar.update(' 70.00M 7% 1.47MB/s 0:00:02 (xfr#2, to-chk=99/1000)')
assert capsys.readouterr().err.endswith(' 2.0MB/s ETA 0:49:30\033[K')
def test_eta_unknown_within_big_file(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 10.0, 50.0]).__next__)
bar = sb._ProgressBar()
assert bar.update(' 10.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=10/20)')
# No file finished in the last 30 s
assert bar.update(' 50.00M 5% 1.47MB/s 0:00:02')
assert capsys.readouterr().err.endswith(' 50.00M copied 1.0MB/s ETA --:--:--\033[K')
def test_estimate_while_scanning(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=4000) bar = sb._ProgressBar(estimate=4000)
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1000/2000)') assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1000/2000)')
assert bar.total is None assert bar.total is None
assert capsys.readouterr().err == \ assert capsys.readouterr().err == \
'\r[#######-----------------------] ~25% 1,000/~4,000 files 2.00M copied ETA ~0:01:30\033[K' '\r[#######-----------------------] ~25% 1,000/~4,000 files 2.00M copied 66.7KB/s ETA ~0:01:30\033[K'
def test_estimate_exceeded(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: def test_estimate_exceeded(self, capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
bar = sb._ProgressBar(estimate=1000) bar = sb._ProgressBar(estimate=1000)
# More files than in the last backup: estimate from the files found so far # More files than in the last backup: estimate from the files found so far
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=500/2000)') assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=500/2000)')
assert capsys.readouterr().err == \ assert capsys.readouterr().err == \
'\r[######################--------] ~75% 1,500/~2,000 files 2.00M copied ETA ~0:00:10\033[K' '\r[######################--------] ~75% 1,500/~2,000 files 2.00M copied 66.7KB/s ETA ~0:00:10\033[K'
def test_estimate_capped_while_scanning(self, monkeypatch: pytest.MonkeyPatch, def test_estimate_capped_while_scanning(self, capsys: pytest.CaptureFixture[str]) -> None:
capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
bar = sb._ProgressBar(estimate=1000) bar = sb._ProgressBar(estimate=1000)
# All files found so far are checked, but rsync may still find more # All files found so far are checked, but rsync may still find more
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)') assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)')
assert bar.total is None assert bar.total is None
assert ' ~99% 2,000/~2,000 files 2.00M copied ETA ~0:00:00\033[K' in capsys.readouterr().err assert ' ~99% 2,000/~2,000 files 2.00M copied 66.7KB/s ETA ~0:00:00\033[K' in capsys.readouterr().err
def test_no_estimate_without_last_count(self, capsys: pytest.CaptureFixture[str]) -> None: def test_no_estimate_without_last_count(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar() bar = sb._ProgressBar()
# The files found so far are only a guess at the total near the end of the scan # The files found so far are only a guess at the total near the end of the scan
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)') assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)')
assert capsys.readouterr().err == '\r2,000 files checked 2.00M copied (scanning)\033[K' assert capsys.readouterr().err == '\r2,000 files checked 2.00M copied 66.7KB/s (scanning)\033[K'
def test_real_total_replaces_estimate(self, capsys: pytest.CaptureFixture[str]) -> None: def test_real_total_replaces_estimate(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=4000) bar = sb._ProgressBar(estimate=4000)