Compare commits

...
7 Commits
Author SHA1 Message Date
Fuxino 13cae98776 Bump version to 4.4.0 2026-10-14 09:48:18 +02:00
Fuxino 7cd90f8d11 Add a progress bar for rsync
On a terminal, run rsync with --info=progress2 and draw its updates as a
progress bar on stderr. The ETA is estimated locally, since rsync reports
the elapsed time when a file finishes. Disable it with --no-progress.
2026-10-14 09:46:50 +02:00
Fuxino d0fb5b65fb Read rsync output while it runs
Read the output as it arrives, split on newlines and carriage returns,
instead of all at once with communicate(). This is needed for the
progress bar. Invalid UTF-8 in the output is replaced instead of raising.
2026-10-14 09:42:37 +02:00
Fuxino a3ace84c8b Log each old backup as it is removed
Removing old backups can take minutes on a slow drive; logging each one with
its position (e.g. "(1/3)") shows that the backup is still progressing.
2026-10-14 09:40:35 +02:00
Fuxino 10eccd360c Add mypy configuration
Check both the package and the tests by default, require type hints on all functions and ignore the missing stubs for paramiko, systemd and dbus.
2026-10-14 08:39:46 +02:00
Fuxino 0829c33053 Fix mypy errors in simple_backup() and Backup
Accept None for the missing inputs, output, exclude and keep values and replace them with their defaults in __init__, and annotate the SSH client, which mypy typed as None and so skipped checking the remote code.
2026-10-14 08:37:06 +02:00
Fuxino cfe3b28436 Fix mypy errors in the tests
The config values are now a TypedDict, so check inputs for None before sorting, use literal keys and use ConfigArgs as the fake _read_config() return type.
2026-10-14 08:37:06 +02:00
8 changed files with 259 additions and 45 deletions

No files matched your search

+7 -1
View File
@@ -1,4 +1,4 @@
.TH SIMPLE_BACKUP 1 2026-10-08 SIMPLE_BACKUP 4.3.0
.TH SIMPLE_BACKUP 1 2026-10-14 SIMPLE_BACKUP 4.4.0
.SH NAME
simple_backup \- Backup files and folders using rsync
.SH SYNOPSIS
@@ -105,6 +105,12 @@ according to \-\-keep will be left, and the new backup may be incomplete. In par
.B \-\-no\-syslog
Don't use systemd journal for logging.
.TP
.B \-\-no\-progress
Don't show a progress bar while rsync copies the files. The progress bar is only shown when
the output is a terminal, so it never appears in the systemd journal or in redirected output.
While rsync is still looking for files, the bar shows \(aq(scanning)\(aq, and the percentage
and estimated time are only approximate.
.TP
.B \-\-rsync\-options OPTIONS [OPTION...]
By default, the following rsync options are used:
.RS
+10
View File
@@ -47,6 +47,16 @@ packages = ["simple_backup"]
[tool.setuptools.dynamic]
version = { attr = "simple_backup.__version__" }
[tool.mypy]
files = ["simple_backup", "tests"]
disallow_untyped_defs = true
warn_unused_ignores = true
# No usable stubs (types-paramiko lags behind paramiko 5)
[[tool.mypy.overrides]]
module = ["paramiko", "systemd", "dbus"]
ignore_missing_imports = true
[tool.pytest.ini_options]
testpaths = ["tests"]
# Import the repo copy, not an installed one
+1 -1
View File
@@ -1,3 +1,3 @@
"""Init."""
__version__ = '4.3.0'
__version__ = '4.4.0'
+129 -30
View File
@@ -18,10 +18,12 @@ from __future__ import annotations
import sys
import os
import pwd
from typing import Callable, List, Optional, ParamSpec, TypedDict, TypeVar, Union
import io
import re
from typing import Callable, Iterator, List, Optional, ParamSpec, TypedDict, TypeVar
import warnings
from functools import wraps
from shutil import rmtree, which
from shutil import get_terminal_size, rmtree, which
import shlex
import argparse
import configparser
@@ -72,6 +74,10 @@ R = TypeVar('R')
# Seconds to wait for the TCP connection to the SSH server
SSH_TIMEOUT = 15
# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
PROGRESS_RE = re.compile(r'\s*(\S+)\s+(\d+)%\s+(\S+)\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=\d+/\d+\))?\s*')
PROGRESS_BAR_WIDTH = 30
def timing(func: Callable[P, R]) -> Callable[P, R]:
"""Decorator to measure execution time of a function
@@ -98,6 +104,59 @@ class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFo
"""Custom format for argparse help text"""
class _ProgressBar:
"""Draws rsync --info=progress2 updates as a bar on stderr"""
def __init__(self) -> None:
self._start = default_timer()
# rsync reports ir-chk while incremental recursion is still finding files
self._scanning = True
self._shown = False
def update(self, record: str) -> bool:
"""Draw the bar if record is a progress update, and return whether it was"""
m = PROGRESS_RE.fullmatch(record)
if m is None:
return False
size, percent, speed, chk = m.group(1), int(m.group(2)), m.group(3), m.group(4)
if chk is not None:
self._scanning = chk == 'ir'
# rsync's own time field is the elapsed time on records ending a file, so estimate the ETA here
if 0 < percent < 100:
eta = int((default_timer() - self._start) * (100 - percent) / percent)
text = f'{percent:3d}% {size} {speed} ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
else:
text = f'{percent:3d}% {size} {speed}'
if self._scanning:
text += ' (scanning)'
width = get_terminal_size().columns - 1
# Fixed width, so the bar doesn't jump when the text changes length
if len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
filled = PROGRESS_BAR_WIDTH * percent // 100
text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}] {text}'
sys.stderr.write(f'\r{text[:width]}\033[K')
sys.stderr.flush()
self._shown = True
return True
def clear(self) -> None:
"""Remove the bar, so that log messages start on a clean line"""
if self._shown:
sys.stderr.write('\r\033[K')
sys.stderr.flush()
class Backup:
"""Main class defining parameters and functions for performing backup
@@ -122,6 +181,8 @@ class Backup:
Run remote rsync with sudo
remove_before: bool
Indicate if removing old backups will be performed before copying files
progress: bool
Show a progress bar while rsync runs
Methods:
check_params():
@@ -136,40 +197,44 @@ class Backup:
Perform the backup
"""
def __init__(self, inputs: List[str], output: str, exclude: List[str], keep: int, options: str,
def __init__(self, inputs: Optional[List[str]], output: Optional[str], exclude: Optional[List[str]],
keep: Optional[int], options: str,
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None,
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False) -> None:
self.inputs = inputs
self.output = output
self.exclude = exclude
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False,
progress: bool = False) -> None:
# Missing values are reported by check_params()
self.inputs = inputs if inputs is not None else []
self.output = output if output is not None else ''
self.exclude = exclude if exclude is not None else []
self.options = options
self.keep = keep
self.keep = keep if keep is not None else -1
self.ssh_host = ssh_host
self.ssh_user = ssh_user
self.ssh_keyfile = ssh_keyfile
self.remote_sudo = remote_sudo
self._remove_before = remove_before
self._verbose = verbose
self._progress = progress
self._last_backup = ''
self._server = ''
self._output_dir = ''
self._inputs_path = ''
self._exclude_path = ''
self._remote = False
self._ssh = None
self._ssh: Optional[paramiko.client.SSHClient] = None
self._password_auth = False
self._removed_count = 0
def check_params(self, homedir: str = '') -> int:
"""Check if parameters for the backup are valid"""
if self.inputs is None or len(self.inputs) == 0:
if len(self.inputs) == 0:
logger.info(
'No existing files or directories specified for backup. Nothing to do')
return 1
if self.output is None:
if self.output == '':
logger.critical(
'No output path specified. Use -o argument or specify output path in configuration file')
@@ -207,9 +272,6 @@ class Backup:
self.output = os.path.abspath(self.output)
if self.keep is None:
self.keep = -1
return 0
def _resolve_remote_path(self, path: str) -> Optional[str]:
@@ -279,7 +341,11 @@ class Backup:
logger.info('Removing old backups...')
dirs.sort()
for i in range(n_backup - self.keep):
n_remove = n_backup - self.keep
for i in range(n_remove):
logger.info('Removing backup %s (%d/%d)...', dirs[i], i + 1, n_remove)
if self.remote_sudo:
_, _, stderr = self._ssh.exec_command(
f'sudo rm -r {shlex.quote(f"{self.output}/simple_backup/{dirs[i]}")}')
@@ -318,7 +384,11 @@ class Backup:
logger.info('Removing old backups...')
dirs.sort()
for i in range(n_backup - self.keep):
n_remove = n_backup - self.keep
for i in range(n_remove):
logger.info('Removing backup %s (%d/%d)...', dirs[i], i + 1, n_remove)
try:
rmtree(f'{self.output}/simple_backup/{dirs[i]}')
count += 1
@@ -605,10 +675,9 @@ class Backup:
fd, self._exclude_path = mkstemp(prefix='tmp_exclude', text=True)
with os.fdopen(fd, 'w', encoding='utf-8') as fp:
if self.exclude is not None:
for e in self.exclude:
fp.write(e)
fp.write('\n')
for e in self.exclude:
fp.write(e)
fp.write('\n')
if self.keep != -1 and self._remove_before:
self.remove_old_backups()
@@ -640,25 +709,39 @@ class Backup:
if self._remote and self.remote_sudo:
args.append('--rsync-path=sudo rsync')
bar = None
if self._progress:
args.append('--info=progress2')
bar = _ProgressBar()
with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p:
output: Union[bytes, List[str]]
output, _ = p.communicate()
assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
p.stdin.close()
output = []
try:
for r in _read_records(p.stdout):
if r != '' and (bar is None or not bar.update(r)):
output.append(r)
finally:
if bar is not None:
bar.clear()
try:
del os.environ['SSHPASS']
except KeyError:
pass
returncode = p.returncode
output = output.decode("utf-8").split('\n')
returncode = p.wait()
if returncode == 0:
if self._verbose:
logger.info('rsync: %s', output)
else:
logger.info('rsync: %s', output[-3])
logger.info('rsync: %s', output[-2])
logger.info('rsync: %s', output[-1])
else:
self._returncode_log(returncode)
@@ -703,6 +786,21 @@ class Backup:
return 0
def _read_records(stream: io.BufferedIOBase) -> Iterator[str]:
"""Yield rsync output as it arrives, split on '\\n' and '\\r' (which ends progress updates)"""
buffer = b''
while chunk := stream.read1(65536):
*records, buffer = re.split(rb'[\r\n]', buffer + chunk)
for r in records:
yield r.decode('utf-8', errors='replace')
if buffer != b'':
yield buffer.decode('utf-8', errors='replace')
def _detect_user() -> Optional[str]:
if os.geteuid() == 0:
return os.getenv('SUDO_USER') or os.getenv('DOAS_USER')
@@ -743,6 +841,8 @@ def _parse_arguments() -> argparse.Namespace:
help='Remove old backups before executing the backup, instead of after')
parser.add_argument('--no-syslog', action='store_true',
help='Disable systemd journal logging')
parser.add_argument('--no-progress', action='store_true',
help='Don\'t show a progress bar (it is only shown on a terminal)')
parser.add_argument('--rsync-options', nargs='+',
choices=['a', 'l', 'p', 't', 'g', 'o',
'c', 'h', 's', 'D', 'H', 'X'],
@@ -1015,10 +1115,9 @@ def simple_backup() -> int:
if args.numeric_ids or config_args['numeric_ids']:
rsync_options.append('--numeric-ids')
rsync_options = ' '.join(rsync_options)
backup = Backup(inputs, output, exclude, keep, rsync_options, ssh_host, ssh_user, ssh_keyfile,
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose)
backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile,
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose,
progress=not args.no_progress and sys.stderr.isatty())
try:
return_code = backup.check_params(homedir)
+15 -2
View File
@@ -27,7 +27,7 @@ class FakeBackup:
def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str,
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None,
ssh_keyfile: Optional[str] = None, remote_sudo: bool = False,
remove_before: bool = False, verbose: bool = False) -> None:
remove_before: bool = False, verbose: bool = False, progress: bool = False) -> None:
self.inputs = inputs
self.output = output
self.exclude = exclude
@@ -39,6 +39,7 @@ class FakeBackup:
self.remote_sudo = remote_sudo
self.remove_before = remove_before
self.verbose = verbose
self.progress = progress
self.homedir: Optional[str] = None
self.ran = False
self.closed = False
@@ -91,7 +92,7 @@ def config_calls(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, Optional[st
calls: list[tuple[str, Optional[str]]] = []
read_config = sb._read_config
def record(config_file: str, user: Optional[str] = None) -> dict[str, Any]:
def record(config_file: str, user: Optional[str] = None) -> sb.ConfigArgs:
calls.append((config_file, user))
return read_config('/nonexistent/simple_backup.conf', user)
@@ -224,6 +225,18 @@ ssh_keyfile=/keys/id
b = last_backup()
assert b.remote_sudo and b.remove_before and b.verbose
@pytest.mark.parametrize('tty, argv, progress', [
(True, [], True),
(True, ['--no-progress'], False),
(False, [], False),
])
def test_progress(self, run_cli: RunCli, config: Callable[[str], str], monkeypatch: pytest.MonkeyPatch,
tty: bool, argv: list[str], progress: bool) -> None:
monkeypatch.setattr(sb.sys.stderr, 'isatty', lambda: tty)
assert run_cli('-c', config('backup_dir=/backups'), *argv) == 0
assert last_backup().progress is progress
def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None:
FakeBackup.check_code = 2
+4 -3
View File
@@ -85,6 +85,7 @@ keep=3
config_args = sb._read_config(config_file, USER)
assert config_args['inputs'] is not None
assert sorted(config_args['inputs']) == [f'{tmp_path}/one', f'{tmp_path}/two']
# Expanded later, as it may be a path on the server
assert config_args['output'] == '~/backups'
@@ -176,9 +177,9 @@ numeric_ids=
config_args = sb._read_config(config_file, USER)
for key in ['ssh_host', 'ssh_user', 'ssh_keyfile']:
assert config_args[key] is None
assert config_args['ssh_host'] is None
assert config_args['ssh_user'] is None
assert config_args['ssh_keyfile'] is None
assert config_args['remote_sudo'] is False
assert config_args['numeric_ids'] is False
+89 -6
View File
@@ -1,5 +1,6 @@
"""Tests for local backups, with real rsync runs and with a fake Popen"""
import io
import os
from datetime import datetime, timedelta
from pathlib import Path
@@ -52,8 +53,9 @@ def out(tmp_path: Path) -> Path:
def make_backup(out: Path, inputs: list[str], keep: int = -1, exclude: Optional[list[str]] = None,
remove_before: bool = False) -> sb.Backup:
backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before)
remove_before: bool = False, progress: bool = False) -> sb.Backup:
backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before,
progress=progress)
assert backup.check_params() == 0
@@ -79,14 +81,14 @@ class TestCheckParams:
assert sb.Backup([], str(out), [], -1, OPTIONS).check_params() == 1
def test_no_output(self) -> None:
assert sb.Backup(['/x'], None, [], -1, OPTIONS).check_params() == 2 # type: ignore[arg-type]
assert sb.Backup(['/x'], None, [], -1, OPTIONS).check_params() == 2
def test_missing_output(self, tmp_path: Path) -> None:
assert sb.Backup(['/x'], str(tmp_path / 'missing'), [], -1, OPTIONS).check_params() == 2
def test_relative_output(self, out: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.chdir(out.parent)
backup = sb.Backup(['/x'], out.name, [], None, OPTIONS) # type: ignore[arg-type]
backup = sb.Backup(['/x'], out.name, [], None, OPTIONS)
assert backup.check_params() == 0
assert backup.output == str(out)
@@ -140,6 +142,15 @@ class TestRsync:
assert backups(out) == remaining
def test_removal_logged(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture) -> None:
for _ in range(3):
assert run_backup(out, [str(src)]) == 0
assert run_backup(out, [str(src)], keep=0) == 0
assert 'Removing backup 2026-10-14_10-00-00 (1/3)...' in caplog.text
assert 'Removing backup 2026-10-14_12-00-00 (3/3)...' in caplog.text
assert 'Removed 3 backups' in caplog.text
def test_remove_before(self, src: Path, out: Path) -> None:
assert run_backup(out, [str(src)]) == 0
assert run_backup(out, [str(src)], keep=0, remove_before=True) == 0
@@ -177,6 +188,16 @@ class TestRsync:
assert run_backup(out, [str(src)], keep=0) == 0
assert backups(out) == ['2026-10-14_12-00-00']
def test_progress(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture,
capsys: pytest.CaptureFixture[str]) -> None:
assert run_backup(out, [str(src)], progress=True) == 0
assert '100%' in capsys.readouterr().err
# Only the summary is logged, not the progress updates
assert 'rsync: sent' in caplog.text
assert 'rsync: total size' in caplog.text
assert '%' not in caplog.text
def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None:
config_file = str(tmp_path / 'missing.conf')
@@ -186,6 +207,66 @@ class TestRsync:
assert backups(out) == ['2026-10-14_11-00-00']
class ChunkStream(io.BufferedIOBase):
"""Returns the given chunks one per read1() call, like a pipe"""
def __init__(self, chunks: list[bytes]) -> None:
self.chunks = chunks
def read1(self, size: int = -1) -> bytes:
return self.chunks.pop(0) if self.chunks else b''
class TestReadRecords:
def test_split_on_newline_and_cr(self) -> None:
stream = ChunkStream([b'file\n\r 1.00M 50%\r 2.00M 100%\n\nsent 1 bytes\ntotal'])
assert list(sb._read_records(stream)) == ['file', '', ' 1.00M 50%', ' 2.00M 100%', '', 'sent 1 bytes',
'total']
def test_records_across_chunks(self) -> None:
stream = ChunkStream([b'fi', b'le\r', b'\nna', 'mé\n'.encode(), b'\xff\n'])
assert list(sb._read_records(stream)) == ['file', '', 'namé', '�']
class TestProgressBar:
@pytest.fixture(autouse=True)
def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24)))
# 30 s elapsed at each update
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=2/4)')
assert capsys.readouterr().err == \
'\r[#######-----------------------] 25% 2.00M 1.47MB/s ETA 0:01:30 (scanning)\033[K'
def test_without_human_readable(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
assert bar.update(' 4,000,000 50% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)')
assert capsys.readouterr().err.endswith('] 50% 4,000,000 2.97MB/s ETA 0:00:30\033[K')
def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
for record in ['home/user/file 50% 1:00:00', 'sent 1 bytes received 2 bytes', 'total size is 3']:
assert not bar.update(record)
bar.clear()
assert capsys.readouterr().err == ''
def test_narrow_terminal(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((20, 24)))
assert sb._ProgressBar().update(' 6.00M 100% 1.48MB/s 0:00:03 (xfr#3, to-chk=0/4)')
# No room for the bar
assert capsys.readouterr().err == '\r100% 6.00M 1.48MB\033[K'
class FakePopen:
"""Stands in for rsync: creates the destination and returns a chosen return code"""
@@ -195,6 +276,8 @@ class FakePopen:
def __init__(self, args: list[str], **kwargs: Any) -> None:
FakePopen.calls.append(args)
os.makedirs(args[args.index('/') + 1])
self.stdin = io.BytesIO()
self.stdout = io.BytesIO(b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n')
def __enter__(self) -> 'FakePopen':
return self
@@ -202,8 +285,8 @@ class FakePopen:
def __exit__(self, *args: Any) -> None:
pass
def communicate(self) -> tuple[bytes, None]:
return b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n', None
def wait(self) -> int:
return FakePopen.returncode
@pytest.fixture
+4 -2
View File
@@ -257,6 +257,8 @@ class RecordPopen:
def __init__(self, args: list[str], **kwargs: Any) -> None:
RecordPopen.calls.append(args)
self.stdin = io.BytesIO()
self.stdout = io.BytesIO(b'sent 1 bytes received 2 bytes\ntotal size is 3\n')
def __enter__(self) -> 'RecordPopen':
return self
@@ -264,8 +266,8 @@ class RecordPopen:
def __exit__(self, *args: Any) -> None:
pass
def communicate(self) -> tuple[bytes, None]:
return b'sent 1 bytes received 2 bytes\ntotal size is 3\n', None
def wait(self) -> int:
return RecordPopen.returncode
@pytest.mark.usefixtures('connect')