fuxino/simple_backup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Estimate progress while rsync scans from the last backup's file count

Browse Source 
With incremental recursion rsync only knows the total near the end, so the bar showed
no ETA for most of the backup. Save the file count as a third line of the state file
and use it as an estimate until rsync reports the real total.
This commit is contained in:
Fuxino
2026-11-02 20:56:12 +01:00
parent 5e33781974
commit ec27b8d77f
6 changed files with 142 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1
View File
@@ -64,6 +64,7 @@ The only rsync error that is ignored is return code 24 (source files vanished du
## Last backup
After each successful backup, simple_backup saves its time and directory to `~/.local/state/simple_backup/last_backup` (or `$XDG_STATE_HOME/simple_backup/last_backup`).
It also saves the number of files, so that the next run can show progress and an estimated time while rsync is still looking for files.
When run with sudo or doas, the file goes to the home directory of the user who ran the command.
Print it with:
man/simple_backup.1
+6 -3
View File
@@ -108,8 +108,9 @@ Don't use systemd journal for logging.
.B \-\-no\-progress
Don't show a progress bar while rsync copies the files. The progress bar is only shown when
the output is a terminal, so it never appears in the systemd journal or in redirected output.
While rsync is still looking for files, the bar shows \(aq(scanning)\(aq, and the percentage
and estimated time are only approximate.
While rsync is still looking for files, the percentage and estimated time (marked with
\(aq~\(aq) are based on the number of files in the last backup, or the bar shows
\(aq(scanning)\(aq if that is unknown or already exceeded.
.TP
.B \-\-rsync\-options OPTIONS [OPTION...]
By default, the following rsync options are used:
@@ -227,7 +228,9 @@ Bad configuration file or options (for example, only one of SSH host and SSH use
Written after each successful backup (local or remote), replacing the previous one. The first line
is the completion time in ISO 8601 format with the UTC offset (for example
\(aq2026\-10\-25T14:03:12+02:00\(aq), the second line the backup directory (prefixed with
\(aquser@host:\(aq for remote backups). The default location is
\(aquser@host:\(aq for remote backups), and the third line, if present, the number of files, used
for the progress bar estimate (only counted when the progress bar is shown, otherwise the previous
count is kept). The default location is
\fI~/.local/state/simple_backup/last_backup\fR. When running as root with sudo, doas or \-\-user,
the file is written to that user\(aqs \fI~/.local/state\fR and owned by them. Useful for scripts
that remind you when a backup is overdue.
simple_backup/simple_backup.py
+59 -14
View File
@@ -94,14 +94,27 @@ class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFo
class _ProgressBar:
"""Draws rsync --info=progress2 updates as a bar on stderr"""
def __init__(self) -> None:
def __init__(self, estimate: int = 0) -> None:
self._start = default_timer()
# rsync reports ir-chk while incremental recursion is still finding files
self._scanning = True
self._checked = 0
self._total = 0
# Total files of the last backup, used until rsync knows the real total
self._estimate = estimate
self._shown = False
@property
def total(self) -> int | None:
"""Total number of files, once rsync has found them all"""
return None if self._scanning or self._total == 0 else self._total
def _eta(self, total: int) -> str:
eta = int((default_timer() - self._start) * (total - self._checked) / self._checked)
return f'{eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
def update(self, record: str) -> bool:
"""Draw the bar if record is a progress update, and return whether it was"""
@@ -119,20 +132,29 @@ class _ProgressBar:
self._checked = self._total - int(m.group(3))
# Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out
if self._scanning or self._total == 0:
text = f'{self._checked:,} files checked {size} copied (scanning)'
percent = None
percent: int | None
if self.total is None:
# Estimate from the last backup, unless this one already has more files
if 0 < self._checked < self._estimate:
percent = self._checked * 100 // self._estimate
text = f'{f"~{percent}":>3}% {self._checked:,}/~{self._estimate:,} files {size} copied'
if percent > 0:
text += f' ETA ~{self._eta(self._estimate)}'
else:
text = f'{self._checked:,} files checked {size} copied (scanning)'
percent = None
else:
percent = self._checked * 100 // self._total
text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied'
if 0 < percent < 100:
eta = int((default_timer() - self._start) * (self._total - self._checked) / self._checked)
text += f' ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
text += f' ETA {self._eta(self._total)}'
width = get_terminal_size().columns - 1
# Fixed width, so the bar doesn't jump when the text changes length; no bar until the total is known
# Fixed width, so the bar doesn't jump when the text changes length; no bar without a total or estimate
if percent is not None and len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
filled = PROGRESS_BAR_WIDTH * percent // 100
text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}] {text}'
@@ -195,7 +217,7 @@ class Backup:
keep: int | None, options: str,
ssh_host: str | None = None, ssh_user: str | None = None, ssh_keyfile: str | None = None,
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False,
progress: bool = False) -> None:
progress: bool = False, file_estimate: int = 0) -> None:
# Missing values are reported by check_params()
self.inputs = inputs if inputs is not None else []
self.output = output if output is not None else ''
@@ -209,6 +231,9 @@ class Backup:
self._remove_before = remove_before
self._verbose = verbose
self._progress = progress
self._file_estimate = file_estimate
# Files in the backup, known only with the progress bar
self.file_count: int | None = None
self._last_backup = ''
self._server = ''
self._output_dir = ''
@@ -768,7 +793,7 @@ class Backup:
if self._progress:
args.append('--info=progress2')
bar = _ProgressBar()
bar = _ProgressBar(self._file_estimate)
with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False, env=env) as p:
assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
@@ -787,6 +812,9 @@ class Backup:
returncode = p.wait()
if bar is not None:
self.file_count = bar.total
if returncode == 0:
# Without -v, only the summary, which may be fewer than two lines (e.g. with --rsync-options q)
for r in output:
@@ -1105,12 +1133,28 @@ def _state_file(owner: pwd.struct_passwd | None) -> str:
return os.path.join(base, 'simple_backup', 'last_backup')
def _write_state(user: str | None, destination: str) -> None:
"""Record the time and directory of the last successful backup (two lines)"""
def _last_file_count(user: str | None) -> int:
"""Files in the last backup (third line of the state file), or 0 if unknown"""
try:
with open(_state_file(_state_owner(user)), encoding='utf-8') as fp:
return int(fp.read().splitlines()[2])
except (OSError, ValueError, IndexError):
return 0
def _write_state(user: str | None, destination: str, file_count: int | None = None) -> None:
"""Record the time, directory and file count of the last successful backup (one per line)"""
owner = _state_owner(user)
path = _state_file(owner)
tmp = ''
# Without the progress bar the count is unknown, so keep the previous one
if file_count is None:
file_count = _last_file_count(user)
count_line = f'{file_count}\n' if file_count > 0 else ''
euid, egid = os.geteuid(), os.getegid()
# Create the file (and any missing directories) as the user, not root
@@ -1125,7 +1169,7 @@ def _write_state(user: str | None, destination: str) -> None:
fd, tmp = mkstemp(prefix='.last_backup', dir=os.path.dirname(path), text=True)
with os.fdopen(fd, 'w', encoding='utf-8') as fp:
fp.write(f'{datetime.now().astimezone().isoformat(timespec="seconds")}\n{destination}\n')
fp.write(f'{datetime.now().astimezone().isoformat(timespec="seconds")}\n{destination}\n{count_line}')
os.replace(tmp, path)
except OSError as e:
@@ -1260,9 +1304,10 @@ def simple_backup() -> int:
if args.numeric_ids or config_args['numeric_ids']:
rsync_options.append('--numeric-ids')
progress = not args.no_progress and sys.stderr.isatty()
backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile,
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose,
progress=not args.no_progress and sys.stderr.isatty())
progress=progress, file_estimate=_last_file_count(user) if progress else 0)
try:
return_code = backup.check_params(homedir)
@@ -1273,7 +1318,7 @@ def simple_backup() -> int:
return_code = backup.run()
if return_code == 0:
_write_state(user, backup.destination)
_write_state(user, backup.destination, backup.file_count)
return return_code
finally:
tests/test_cli.py
+16 -2
View File
@@ -32,7 +32,8 @@ class FakeBackup:
def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str,
ssh_host: str | None = None, ssh_user: str | None = None,
ssh_keyfile: str | None = None, remote_sudo: bool = False,
remove_before: bool = False, verbose: bool = False, progress: bool = False) -> None:
remove_before: bool = False, verbose: bool = False, progress: bool = False,
file_estimate: int = 0) -> None:
self.inputs = inputs
self.output = output
self.exclude = exclude
@@ -45,6 +46,8 @@ class FakeBackup:
self.remove_before = remove_before
self.verbose = verbose
self.progress = progress
self.file_estimate = file_estimate
self.file_count: int | None = None
self.homedir: str | None = None
self.ran = False
self.closed = False
@@ -87,7 +90,7 @@ def as_root(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(os, 'geteuid', lambda: 0)
# Writing the state file as root would call seteuid() (covered in test_state.py)
monkeypatch.setattr(sb, '_write_state', lambda user, destination: None)
monkeypatch.setattr(sb, '_write_state', lambda user, destination, file_count=None: None)
for var in ['SUDO_USER', 'SUDO_UID', 'DOAS_USER']:
monkeypatch.delenv(var, raising=False)
@@ -266,6 +269,17 @@ ssh_keyfile=/keys/id
assert run_cli('-c', config('backup_dir=/backups'), *argv) == 0
assert last_backup().progress is progress
@pytest.mark.parametrize('tty, estimate', [(True, 1234), (False, 0)])
def test_file_estimate(self, run_cli: RunCli, config: Callable[[str], str], monkeypatch: pytest.MonkeyPatch,
tmp_path: Path, tty: bool, estimate: int) -> None:
monkeypatch.setattr(sb.sys.stderr, 'isatty', lambda: tty)
state = tmp_path / 'state' / 'simple_backup' / 'last_backup'
state.parent.mkdir(parents=True)
state.write_text('2026-10-25T10:00:00+02:00\n/backups/old\n1234\n')
assert run_cli('-c', config('backup_dir=/backups')) == 0
assert last_backup().file_estimate == estimate
def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None:
FakeBackup.check_code = 2
tests/test_local.py
+37
View File
@@ -207,6 +207,19 @@ class TestRsync:
assert 'rsync: total size' in caplog.text
assert '%' not in caplog.text
def test_file_count(self, src: Path, out: Path) -> None:
backup = make_backup(out, [str(src)], progress=True)
assert backup.run() == 0
# The source folder, its two files and the implied parent folders
assert backup.file_count is not None and backup.file_count >= 3
def test_no_file_count_without_progress(self, src: Path, out: Path) -> None:
backup = make_backup(out, [str(src)])
assert backup.run() == 0
assert backup.file_count is None
def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None:
config_file = str(tmp_path / 'missing.conf')
@@ -280,6 +293,30 @@ class TestProgressBar:
assert bar.update(' 3.00M 2% 1.47MB/s 0:00:03')
assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied ETA 0:00:30\033[K')
def test_estimate_while_scanning(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((100, 24)))
bar = sb._ProgressBar(estimate=4000)
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1000/2000)')
assert bar.total is None
assert capsys.readouterr().err == \
'\r[#######-----------------------] ~25% 1,000/~4,000 files 2.00M copied ETA ~0:01:30\033[K'
def test_estimate_exceeded(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=1000)
# More files than in the last backup: back to plain scanning
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=500/2000)')
assert capsys.readouterr().err == '\r1,500 files checked 2.00M copied (scanning)\033[K'
def test_real_total_replaces_estimate(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar(estimate=4000)
assert bar.update(' 4,000,000 1% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)')
assert bar.total == 4
assert ' 75% 3/4 files' in capsys.readouterr().err
def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
tests/test_state.py
+23
View File
@@ -49,6 +49,29 @@ def test_overwrite(state: Path) -> None:
assert state.read_text().splitlines()[1] == DESTINATION
def test_file_count(state: Path) -> None:
sb._write_state(None, DESTINATION, 1234)
assert state.read_text().splitlines()[2] == '1234'
assert sb._last_file_count(None) == 1234
def test_file_count_kept_when_unknown(state: Path) -> None:
write_state(state, '2026-10-01T10:00:00+02:00\n/old\n1234\n')
sb._write_state(None, DESTINATION)
assert state.read_text().splitlines()[1:] == [DESTINATION, '1234']
@pytest.mark.parametrize('text', [None, '2026-10-01T10:00:00+02:00\n/old\n', '2026-10-01T10:00:00+02:00\n/old\nx\n'])
def test_no_file_count(state: Path, text: str | None) -> None:
if text is not None:
write_state(state, text)
assert sb._last_file_count(None) == 0
def test_relative_xdg_state_home_ignored(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
monkeypatch.setenv('XDG_STATE_HOME', 'relative')
monkeypatch.setenv('HOME', str(tmp_path / 'home'))