Compare commits

..
3 Commits
Author SHA1 Message Date
Fuxino fea2e0456a Bump version to 4.7.0 2026-11-21 17:05:46 +01:00
Fuxino 8aef768a23 Base the progress ETA on the last 30 seconds and show the copy speed
The whole-run file rate is dominated by hard-linked files, so after a big
folder moved the ETA showed seconds while gigabytes were still to be copied.
2026-11-21 17:04:49 +01:00
Fuxino efdff589f7 Update PKGBUILD 2026-11-14 11:09:14 +01:00
6 changed files with 131 additions and 24 deletions

No files matched your search

+2 -2
View File
@@ -4,7 +4,7 @@
pkgname=simple_backup
pkgdesc='Simple backup script that uses rsync to copy files'
pkgver=4.6.3
pkgver=4.6.4
pkgrel=1
url="https://git.shouldnt.work/fuxino/$pkgname"
arch=('any')
@@ -24,7 +24,7 @@ checkdepends=('python-pytest'
conflicts=('simple_backup-git')
source=("git+$url?signed#tag=$pkgver")
validpgpkeys=('7E12BC1FF3B6EDB2CD8053EB981A2B2A3BBF5514')
sha256sums=('b1e7802fcd5e615c92f6007238b3699d69e3005e140d87a742283ef1cc6bd37f')
sha256sums=('9e7a1271e5997af952121fe5c6448db2b9fd0898cdd6dba65f1882fc0085aca1')
build()
{
+2 -1
View File
@@ -59,7 +59,8 @@ In that case the user who ran the command is detected automatically, so that '~'
When running as root in other ways (e.g. from a systemd service), specify the user with the --user (or -u) argument.
## Progress
When run in a terminal, simple_backup shows a progress bar with the percentage of files copied and an estimated time to completion.
When run in a terminal, simple_backup shows a progress bar with the percentage of files copied, the copy speed and an estimated time to completion.
Speed and estimated time follow the last 30 seconds, so the estimate adapts when big files are being copied.
Disable it with --no-progress.
## Old backups
+4 -1
View File
@@ -1,4 +1,4 @@
.TH SIMPLE_BACKUP 1 2026-11-14 SIMPLE_BACKUP 4.6.4
.TH SIMPLE_BACKUP 1 2026-11-21 SIMPLE_BACKUP 4.7.0
.SH NAME
simple_backup \- Backup files and folders using rsync
.SH SYNOPSIS
@@ -116,6 +116,9 @@ Don't use systemd journal for logging.
.B \-\-no\-progress
Don't show a progress bar while rsync copies the files. The progress bar is only shown when
the output is a terminal, so it never appears in the systemd journal or in redirected output.
It shows the percentage of files copied, the copy speed and the estimated time, both from the
last 30 seconds; the time shows \(aq\-\-:\-\-:\-\-\(aq when no file finished in that time (e.g. while
copying a big file).
While rsync is still looking for files, the percentage and estimated time (marked with
\(aq~\(aq) are based on the number of files in the last backup (or on the files found so far,
once this backup has more), and stay below 100%. Without a previous count the bar shows
+1 -1
View File
@@ -1,3 +1,3 @@
"""Init."""
__version__ = '4.6.4'
__version__ = '4.7.0'
+75 -4
View File
@@ -74,6 +74,10 @@ SSH_TIMEOUT = 15
# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
PROGRESS_RE = re.compile(r'\s*(\S+)\s+\d+%\s+\S+\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=(\d+)/(\d+)\))?\s*')
PROGRESS_BAR_WIDTH = 30
# rsync's transferred size: 1,234B, 1.23MB (-h), 1.17MiB (-hh) or 1.23M (older rsync)
SIZE_RE = re.compile(r'([\d.,]+)([KMGTP]?)(i?)B?')
# Seconds of recent progress the ETA and speed come from, so they follow changes of pace
RATE_WINDOW = 30
def _format_duration(seconds: float) -> str:
@@ -87,6 +91,43 @@ def _format_duration(seconds: float) -> str:
return f'{s // 60:02d}:{s % 60:02d}'
def _parse_size(size: str) -> int | None:
"""Bytes in an rsync size field, or None if it can't be parsed"""
m = SIZE_RE.fullmatch(size)
if m is None:
return None
number, unit, binary = m.groups()
try:
if unit == '':
return int(number.replace(',', '').replace('.', ''))
# Suffixed sizes use the locale's decimal point
return int(float(number.replace(',', '.')) * (1024 if binary else 1000) ** ('KMGTP'.index(unit) + 1))
except ValueError:
return None
def _format_speed(speed: float) -> str:
"""Format bytes per second like rsync -h"""
if speed < 1000:
return f'{speed:.0f}B/s'
units = ['KB', 'MB', 'GB', 'TB']
i = 0
speed /= 1000
while speed >= 1000 and i < len(units) - 1:
speed /= 1000
i += 1
return f'{speed:.1f}{units[i]}/s'
class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
"""Custom format for argparse help text"""
@@ -103,6 +144,9 @@ class _ProgressBar:
# Total files of the last backup, used until rsync knows the real total
self._estimate = estimate
self._shown = False
# (time, files checked, bytes copied) about once a second, back to the start of the rate window
self._samples: deque[tuple[float, int, int]] = deque([(self._start, 0, 0)])
self._file_rate: float | None = None
@property
def total(self) -> int | None:
@@ -110,8 +154,33 @@ class _ProgressBar:
return None if self._scanning or self._total == 0 else self._total
def _sample(self, copied: int | None) -> str:
"""Update the file rate over the window, and return the copy speed text (empty if unknown)"""
now = default_timer()
if now - self._samples[-1][0] >= 1:
self._samples.append((now, self._checked, copied or 0))
# Keep the newest sample at or before the window start as the base
while len(self._samples) > 1 and self._samples[1][0] <= now - RATE_WINDOW:
self._samples.popleft()
t0, checked0, copied0 = self._samples[0]
span = now - t0
self._file_rate = (self._checked - checked0) / span if span > 0 else None
if copied is None or span <= 0:
return ''
return f' {_format_speed(max(copied - copied0, 0) / span)}'
def _eta(self, total: int) -> str:
eta = int((default_timer() - self._start) * (total - self._checked) / self._checked)
# No file finished within the window (e.g. while copying a big one): no basis for an estimate
if not self._file_rate:
return '--:--:--'
eta = int((total - self._checked) / self._file_rate)
return f'{eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
@@ -131,6 +200,8 @@ class _ProgressBar:
self._total = int(m.group(4))
self._checked = self._total - int(m.group(3))
speed = self._sample(_parse_size(size))
# Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out
percent: int | None
@@ -141,16 +212,16 @@ class _ProgressBar:
if self._estimate > 0 and self._checked > 0:
# The real total is still unknown, so never claim 100%
percent = min(self._checked * 100 // estimate, 99)
text = f'{f"~{percent}":>3}% {self._checked:,}/~{estimate:,} files {size} copied'
text = f'{f"~{percent}":>3}% {self._checked:,}/~{estimate:,} files {size} copied{speed}'
if percent > 0:
text += f' ETA ~{self._eta(estimate)}'
else:
text = f'{self._checked:,} files checked {size} copied (scanning)'
text = f'{self._checked:,} files checked {size} copied{speed} (scanning)'
percent = None
else:
percent = self._checked * 100 // self._total
text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied'
text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied{speed}'
if 0 < percent < 100:
text += f' ETA {self._eta(self._total)}'
+47 -15
View File
@@ -263,10 +263,26 @@ def test_format_duration(seconds: float, text: str) -> None:
assert sb._format_duration(seconds) == text
@pytest.mark.parametrize(('size', 'size_bytes'), [
('30,000,000B', 30_000_000), ('4,000,000', 4_000_000), ('30.0MB', 30_000_000), ('30,0MB', 30_000_000),
('2.00M', 2_000_000), ('1.5GB', 1_500_000_000), ('28.6MiB', 29_989_273), ('512B', 512), ('?', None),
('1.2.3MB', None),
])
def test_parse_size(size: str, size_bytes: int | None) -> None:
assert sb._parse_size(size) == size_bytes
@pytest.mark.parametrize(('speed', 'text'), [
(0, '0B/s'), (999, '999B/s'), (1500, '1.5KB/s'), (10_500_000, '10.5MB/s'), (2e9, '2.0GB/s'), (3e12, '3.0TB/s'),
])
def test_format_speed(speed: float, text: str) -> None:
assert sb._format_speed(speed) == text
class TestProgressBar:
@pytest.fixture(autouse=True)
def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24)))
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
# 30 s elapsed at each update
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
@@ -274,7 +290,7 @@ class TestProgressBar:
bar = sb._ProgressBar()
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1009/2632)')
assert capsys.readouterr().err == '\r1,623 files checked 2.00M copied (scanning)\033[K'
assert capsys.readouterr().err == '\r1,623 files checked 2.00M copied 66.7KB/s (scanning)\033[K'
def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
@@ -282,7 +298,7 @@ class TestProgressBar:
# With --link-dest rsync's percentage leaves out linked files, so the files are counted instead
assert bar.update(' 4,000,000 1% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)')
assert capsys.readouterr().err == \
'\r[######################--------] 75% 3/4 files 4,000,000 copied ETA 0:00:10\033[K'
'\r[######################--------] 75% 3/4 files 4,000,000 copied 133.3KB/s ETA 0:00:10\033[K'
def test_keeps_counts_within_file(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
@@ -291,43 +307,59 @@ class TestProgressBar:
assert bar.update(' 2.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=2/4)')
assert bar.update(' 3.00M 2% 1.47MB/s 0:00:03')
assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied ETA 0:00:30\033[K')
assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied 100.0KB/s ETA 0:00:30\033[K')
def test_estimate_while_scanning(self, monkeypatch: pytest.MonkeyPatch,
def test_eta_follows_recent_pace(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 100.0, 130.0]).__next__)
bar = sb._ProgressBar()
# Fast while linking unchanged files, then one file in the last 30 s
assert bar.update(' 10.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=100/1000)')
assert capsys.readouterr().err.endswith(' 100.0KB/s ETA 0:00:11\033[K')
assert bar.update(' 70.00M 7% 1.47MB/s 0:00:02 (xfr#2, to-chk=99/1000)')
assert capsys.readouterr().err.endswith(' 2.0MB/s ETA 0:49:30\033[K')
def test_eta_unknown_within_big_file(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 10.0, 50.0]).__next__)
bar = sb._ProgressBar()
assert bar.update(' 10.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=10/20)')
# No file finished in the last 30 s
assert bar.update(' 50.00M 5% 1.47MB/s 0:00:02')
assert capsys.readouterr().err.endswith(' 50.00M copied 1.0MB/s ETA --:--:--\033[K')
def test_estimate_while_scanning(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=4000)
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1000/2000)')
assert bar.total is None
assert capsys.readouterr().err == \
'\r[#######-----------------------] ~25% 1,000/~4,000 files 2.00M copied ETA ~0:01:30\033[K'
'\r[#######-----------------------] ~25% 1,000/~4,000 files 2.00M copied 66.7KB/s ETA ~0:01:30\033[K'
def test_estimate_exceeded(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
def test_estimate_exceeded(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=1000)
# More files than in the last backup: estimate from the files found so far
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=500/2000)')
assert capsys.readouterr().err == \
'\r[######################--------] ~75% 1,500/~2,000 files 2.00M copied ETA ~0:00:10\033[K'
'\r[######################--------] ~75% 1,500/~2,000 files 2.00M copied 66.7KB/s ETA ~0:00:10\033[K'
def test_estimate_capped_while_scanning(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
def test_estimate_capped_while_scanning(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=1000)
# All files found so far are checked, but rsync may still find more
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)')
assert bar.total is None
assert ' ~99% 2,000/~2,000 files 2.00M copied ETA ~0:00:00\033[K' in capsys.readouterr().err
assert ' ~99% 2,000/~2,000 files 2.00M copied 66.7KB/s ETA ~0:00:00\033[K' in capsys.readouterr().err
def test_no_estimate_without_last_count(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
# The files found so far are only a guess at the total near the end of the scan
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=0/2000)')
assert capsys.readouterr().err == '\r2,000 files checked 2.00M copied (scanning)\033[K'
assert capsys.readouterr().err == '\r2,000 files checked 2.00M copied 66.7KB/s (scanning)\033[K'
def test_real_total_replaces_estimate(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=4000)