Compare commits

..
5 Commits
Author SHA1 Message Date
Fuxino 13cae98776 Bump version to 4.4.0 2026-10-14 09:48:18 +02:00
Fuxino 7cd90f8d11 Add a progress bar for rsync
On a terminal, run rsync with --info=progress2 and draw its updates as a
progress bar on stderr. The ETA is estimated locally, since rsync reports
the elapsed time when a file finishes. Disable it with --no-progress.
2026-10-14 09:46:50 +02:00
Fuxino d0fb5b65fb Read rsync output while it runs
Read the output as it arrives, split on newlines and carriage returns,
instead of all at once with communicate(). This is needed for the
progress bar. Invalid UTF-8 in the output is replaced instead of raising.
2026-10-14 09:42:37 +02:00
Fuxino a3ace84c8b Log each old backup as it is removed
Removing old backups can take minutes on a slow drive; logging each one with
its position (e.g. "(1/3)") shows that the backup is still progressing.
2026-10-14 09:40:35 +02:00
Fuxino 10eccd360c Add mypy configuration
Check both the package and the tests by default, require type hints on all functions and ignore the missing stubs for paramiko, systemd and dbus.
2026-10-14 08:39:46 +02:00
7 changed files with 238 additions and 21 deletions

No files matched your search

+7 -1
View File
@@ -1,4 +1,4 @@
.TH SIMPLE_BACKUP 1 2026-10-08 SIMPLE_BACKUP 4.3.0 .TH SIMPLE_BACKUP 1 2026-10-14 SIMPLE_BACKUP 4.4.0
.SH NAME .SH NAME
simple_backup \- Backup files and folders using rsync simple_backup \- Backup files and folders using rsync
.SH SYNOPSIS .SH SYNOPSIS
@@ -105,6 +105,12 @@ according to \-\-keep will be left, and the new backup may be incomplete. In par
.B \-\-no\-syslog .B \-\-no\-syslog
Don't use systemd journal for logging. Don't use systemd journal for logging.
.TP .TP
.B \-\-no\-progress
Don't show a progress bar while rsync copies the files. The progress bar is only shown when
the output is a terminal, so it never appears in the systemd journal or in redirected output.
While rsync is still looking for files, the bar shows \(aq(scanning)\(aq, and the percentage
and estimated time are only approximate.
.TP
.B \-\-rsync\-options OPTIONS [OPTION...] .B \-\-rsync\-options OPTIONS [OPTION...]
By default, the following rsync options are used: By default, the following rsync options are used:
.RS .RS
+10
View File
@@ -47,6 +47,16 @@ packages = ["simple_backup"]
[tool.setuptools.dynamic] [tool.setuptools.dynamic]
version = { attr = "simple_backup.__version__" } version = { attr = "simple_backup.__version__" }
[tool.mypy]
files = ["simple_backup", "tests"]
disallow_untyped_defs = true
warn_unused_ignores = true
# No usable stubs (types-paramiko lags behind paramiko 5)
[[tool.mypy.overrides]]
module = ["paramiko", "systemd", "dbus"]
ignore_missing_imports = true
[tool.pytest.ini_options] [tool.pytest.ini_options]
testpaths = ["tests"] testpaths = ["tests"]
# Import the repo copy, not an installed one # Import the repo copy, not an installed one
+1 -1
View File
@@ -1,3 +1,3 @@
"""Init.""" """Init."""
__version__ = '4.3.0' __version__ = '4.4.0'
+115 -12
View File
@@ -18,10 +18,12 @@ from __future__ import annotations
import sys import sys
import os import os
import pwd import pwd
from typing import Callable, List, Optional, ParamSpec, TypedDict, TypeVar, Union import io
import re
from typing import Callable, Iterator, List, Optional, ParamSpec, TypedDict, TypeVar
import warnings import warnings
from functools import wraps from functools import wraps
from shutil import rmtree, which from shutil import get_terminal_size, rmtree, which
import shlex import shlex
import argparse import argparse
import configparser import configparser
@@ -72,6 +74,10 @@ R = TypeVar('R')
# Seconds to wait for the TCP connection to the SSH server # Seconds to wait for the TCP connection to the SSH server
SSH_TIMEOUT = 15 SSH_TIMEOUT = 15
# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
PROGRESS_RE = re.compile(r'\s*(\S+)\s+(\d+)%\s+(\S+)\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=\d+/\d+\))?\s*')
PROGRESS_BAR_WIDTH = 30
def timing(func: Callable[P, R]) -> Callable[P, R]: def timing(func: Callable[P, R]) -> Callable[P, R]:
"""Decorator to measure execution time of a function """Decorator to measure execution time of a function
@@ -98,6 +104,59 @@ class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFo
"""Custom format for argparse help text""" """Custom format for argparse help text"""
class _ProgressBar:
"""Draws rsync --info=progress2 updates as a bar on stderr"""
def __init__(self) -> None:
self._start = default_timer()
# rsync reports ir-chk while incremental recursion is still finding files
self._scanning = True
self._shown = False
def update(self, record: str) -> bool:
"""Draw the bar if record is a progress update, and return whether it was"""
m = PROGRESS_RE.fullmatch(record)
if m is None:
return False
size, percent, speed, chk = m.group(1), int(m.group(2)), m.group(3), m.group(4)
if chk is not None:
self._scanning = chk == 'ir'
# rsync's own time field is the elapsed time on records ending a file, so estimate the ETA here
if 0 < percent < 100:
eta = int((default_timer() - self._start) * (100 - percent) / percent)
text = f'{percent:3d}% {size} {speed} ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
else:
text = f'{percent:3d}% {size} {speed}'
if self._scanning:
text += ' (scanning)'
width = get_terminal_size().columns - 1
# Fixed width, so the bar doesn't jump when the text changes length
if len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
filled = PROGRESS_BAR_WIDTH * percent // 100
text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}] {text}'
sys.stderr.write(f'\r{text[:width]}\033[K')
sys.stderr.flush()
self._shown = True
return True
def clear(self) -> None:
"""Remove the bar, so that log messages start on a clean line"""
if self._shown:
sys.stderr.write('\r\033[K')
sys.stderr.flush()
class Backup: class Backup:
"""Main class defining parameters and functions for performing backup """Main class defining parameters and functions for performing backup
@@ -122,6 +181,8 @@ class Backup:
Run remote rsync with sudo Run remote rsync with sudo
remove_before: bool remove_before: bool
Indicate if removing old backups will be performed before copying files Indicate if removing old backups will be performed before copying files
progress: bool
Show a progress bar while rsync runs
Methods: Methods:
check_params(): check_params():
@@ -139,7 +200,8 @@ class Backup:
def __init__(self, inputs: Optional[List[str]], output: Optional[str], exclude: Optional[List[str]], def __init__(self, inputs: Optional[List[str]], output: Optional[str], exclude: Optional[List[str]],
keep: Optional[int], options: str, keep: Optional[int], options: str,
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None, ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None,
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False) -> None: remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False,
progress: bool = False) -> None:
# Missing values are reported by check_params() # Missing values are reported by check_params()
self.inputs = inputs if inputs is not None else [] self.inputs = inputs if inputs is not None else []
self.output = output if output is not None else '' self.output = output if output is not None else ''
@@ -152,6 +214,7 @@ class Backup:
self.remote_sudo = remote_sudo self.remote_sudo = remote_sudo
self._remove_before = remove_before self._remove_before = remove_before
self._verbose = verbose self._verbose = verbose
self._progress = progress
self._last_backup = '' self._last_backup = ''
self._server = '' self._server = ''
self._output_dir = '' self._output_dir = ''
@@ -278,7 +341,11 @@ class Backup:
logger.info('Removing old backups...') logger.info('Removing old backups...')
dirs.sort() dirs.sort()
for i in range(n_backup - self.keep): n_remove = n_backup - self.keep
for i in range(n_remove):
logger.info('Removing backup %s (%d/%d)...', dirs[i], i + 1, n_remove)
if self.remote_sudo: if self.remote_sudo:
_, _, stderr = self._ssh.exec_command( _, _, stderr = self._ssh.exec_command(
f'sudo rm -r {shlex.quote(f"{self.output}/simple_backup/{dirs[i]}")}') f'sudo rm -r {shlex.quote(f"{self.output}/simple_backup/{dirs[i]}")}')
@@ -317,7 +384,11 @@ class Backup:
logger.info('Removing old backups...') logger.info('Removing old backups...')
dirs.sort() dirs.sort()
for i in range(n_backup - self.keep): n_remove = n_backup - self.keep
for i in range(n_remove):
logger.info('Removing backup %s (%d/%d)...', dirs[i], i + 1, n_remove)
try: try:
rmtree(f'{self.output}/simple_backup/{dirs[i]}') rmtree(f'{self.output}/simple_backup/{dirs[i]}')
count += 1 count += 1
@@ -638,25 +709,39 @@ class Backup:
if self._remote and self.remote_sudo: if self._remote and self.remote_sudo:
args.append('--rsync-path=sudo rsync') args.append('--rsync-path=sudo rsync')
bar = None
if self._progress:
args.append('--info=progress2')
bar = _ProgressBar()
with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p: with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p:
output: Union[bytes, List[str]] assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
output, _ = p.communicate() p.stdin.close()
output = []
try:
for r in _read_records(p.stdout):
if r != '' and (bar is None or not bar.update(r)):
output.append(r)
finally:
if bar is not None:
bar.clear()
try: try:
del os.environ['SSHPASS'] del os.environ['SSHPASS']
except KeyError: except KeyError:
pass pass
returncode = p.returncode returncode = p.wait()
output = output.decode("utf-8").split('\n')
if returncode == 0: if returncode == 0:
if self._verbose: if self._verbose:
logger.info('rsync: %s', output) logger.info('rsync: %s', output)
else: else:
logger.info('rsync: %s', output[-3])
logger.info('rsync: %s', output[-2]) logger.info('rsync: %s', output[-2])
logger.info('rsync: %s', output[-1])
else: else:
self._returncode_log(returncode) self._returncode_log(returncode)
@@ -701,6 +786,21 @@ class Backup:
return 0 return 0
def _read_records(stream: io.BufferedIOBase) -> Iterator[str]:
"""Yield rsync output as it arrives, split on '\\n' and '\\r' (which ends progress updates)"""
buffer = b''
while chunk := stream.read1(65536):
*records, buffer = re.split(rb'[\r\n]', buffer + chunk)
for r in records:
yield r.decode('utf-8', errors='replace')
if buffer != b'':
yield buffer.decode('utf-8', errors='replace')
def _detect_user() -> Optional[str]: def _detect_user() -> Optional[str]:
if os.geteuid() == 0: if os.geteuid() == 0:
return os.getenv('SUDO_USER') or os.getenv('DOAS_USER') return os.getenv('SUDO_USER') or os.getenv('DOAS_USER')
@@ -741,6 +841,8 @@ def _parse_arguments() -> argparse.Namespace:
help='Remove old backups before executing the backup, instead of after') help='Remove old backups before executing the backup, instead of after')
parser.add_argument('--no-syslog', action='store_true', parser.add_argument('--no-syslog', action='store_true',
help='Disable systemd journal logging') help='Disable systemd journal logging')
parser.add_argument('--no-progress', action='store_true',
help='Don\'t show a progress bar (it is only shown on a terminal)')
parser.add_argument('--rsync-options', nargs='+', parser.add_argument('--rsync-options', nargs='+',
choices=['a', 'l', 'p', 't', 'g', 'o', choices=['a', 'l', 'p', 't', 'g', 'o',
'c', 'h', 's', 'D', 'H', 'X'], 'c', 'h', 's', 'D', 'H', 'X'],
@@ -1014,7 +1116,8 @@ def simple_backup() -> int:
rsync_options.append('--numeric-ids') rsync_options.append('--numeric-ids')
backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile, backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile,
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose) remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose,
progress=not args.no_progress and sys.stderr.isatty())
try: try:
return_code = backup.check_params(homedir) return_code = backup.check_params(homedir)
+14 -1
View File
@@ -27,7 +27,7 @@ class FakeBackup:
def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str, def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str,
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_host: Optional[str] = None, ssh_user: Optional[str] = None,
ssh_keyfile: Optional[str] = None, remote_sudo: bool = False, ssh_keyfile: Optional[str] = None, remote_sudo: bool = False,
remove_before: bool = False, verbose: bool = False) -> None: remove_before: bool = False, verbose: bool = False, progress: bool = False) -> None:
self.inputs = inputs self.inputs = inputs
self.output = output self.output = output
self.exclude = exclude self.exclude = exclude
@@ -39,6 +39,7 @@ class FakeBackup:
self.remote_sudo = remote_sudo self.remote_sudo = remote_sudo
self.remove_before = remove_before self.remove_before = remove_before
self.verbose = verbose self.verbose = verbose
self.progress = progress
self.homedir: Optional[str] = None self.homedir: Optional[str] = None
self.ran = False self.ran = False
self.closed = False self.closed = False
@@ -224,6 +225,18 @@ ssh_keyfile=/keys/id
b = last_backup() b = last_backup()
assert b.remote_sudo and b.remove_before and b.verbose assert b.remote_sudo and b.remove_before and b.verbose
@pytest.mark.parametrize('tty, argv, progress', [
(True, [], True),
(True, ['--no-progress'], False),
(False, [], False),
])
def test_progress(self, run_cli: RunCli, config: Callable[[str], str], monkeypatch: pytest.MonkeyPatch,
tty: bool, argv: list[str], progress: bool) -> None:
monkeypatch.setattr(sb.sys.stderr, 'isatty', lambda: tty)
assert run_cli('-c', config('backup_dir=/backups'), *argv) == 0
assert last_backup().progress is progress
def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None: def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None:
FakeBackup.check_code = 2 FakeBackup.check_code = 2
+87 -4
View File
@@ -1,5 +1,6 @@
"""Tests for local backups, with real rsync runs and with a fake Popen""" """Tests for local backups, with real rsync runs and with a fake Popen"""
import io
import os import os
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
@@ -52,8 +53,9 @@ def out(tmp_path: Path) -> Path:
def make_backup(out: Path, inputs: list[str], keep: int = -1, exclude: Optional[list[str]] = None, def make_backup(out: Path, inputs: list[str], keep: int = -1, exclude: Optional[list[str]] = None,
remove_before: bool = False) -> sb.Backup: remove_before: bool = False, progress: bool = False) -> sb.Backup:
backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before) backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before,
progress=progress)
assert backup.check_params() == 0 assert backup.check_params() == 0
@@ -140,6 +142,15 @@ class TestRsync:
assert backups(out) == remaining assert backups(out) == remaining
def test_removal_logged(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture) -> None:
for _ in range(3):
assert run_backup(out, [str(src)]) == 0
assert run_backup(out, [str(src)], keep=0) == 0
assert 'Removing backup 2026-10-14_10-00-00 (1/3)...' in caplog.text
assert 'Removing backup 2026-10-14_12-00-00 (3/3)...' in caplog.text
assert 'Removed 3 backups' in caplog.text
def test_remove_before(self, src: Path, out: Path) -> None: def test_remove_before(self, src: Path, out: Path) -> None:
assert run_backup(out, [str(src)]) == 0 assert run_backup(out, [str(src)]) == 0
assert run_backup(out, [str(src)], keep=0, remove_before=True) == 0 assert run_backup(out, [str(src)], keep=0, remove_before=True) == 0
@@ -177,6 +188,16 @@ class TestRsync:
assert run_backup(out, [str(src)], keep=0) == 0 assert run_backup(out, [str(src)], keep=0) == 0
assert backups(out) == ['2026-10-14_12-00-00'] assert backups(out) == ['2026-10-14_12-00-00']
def test_progress(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture,
capsys: pytest.CaptureFixture[str]) -> None:
assert run_backup(out, [str(src)], progress=True) == 0
assert '100%' in capsys.readouterr().err
# Only the summary is logged, not the progress updates
assert 'rsync: sent' in caplog.text
assert 'rsync: total size' in caplog.text
assert '%' not in caplog.text
def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None: def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None:
config_file = str(tmp_path / 'missing.conf') config_file = str(tmp_path / 'missing.conf')
@@ -186,6 +207,66 @@ class TestRsync:
assert backups(out) == ['2026-10-14_11-00-00'] assert backups(out) == ['2026-10-14_11-00-00']
class ChunkStream(io.BufferedIOBase):
"""Returns the given chunks one per read1() call, like a pipe"""
def __init__(self, chunks: list[bytes]) -> None:
self.chunks = chunks
def read1(self, size: int = -1) -> bytes:
return self.chunks.pop(0) if self.chunks else b''
class TestReadRecords:
def test_split_on_newline_and_cr(self) -> None:
stream = ChunkStream([b'file\n\r 1.00M 50%\r 2.00M 100%\n\nsent 1 bytes\ntotal'])
assert list(sb._read_records(stream)) == ['file', '', ' 1.00M 50%', ' 2.00M 100%', '', 'sent 1 bytes',
'total']
def test_records_across_chunks(self) -> None:
stream = ChunkStream([b'fi', b'le\r', b'\nna', 'mé\n'.encode(), b'\xff\n'])
assert list(sb._read_records(stream)) == ['file', '', 'namé', '�']
class TestProgressBar:
@pytest.fixture(autouse=True)
def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24)))
# 30 s elapsed at each update
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=2/4)')
assert capsys.readouterr().err == \
'\r[#######-----------------------] 25% 2.00M 1.47MB/s ETA 0:01:30 (scanning)\033[K'
def test_without_human_readable(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
assert bar.update(' 4,000,000 50% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)')
assert capsys.readouterr().err.endswith('] 50% 4,000,000 2.97MB/s ETA 0:00:30\033[K')
def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
for record in ['home/user/file 50% 1:00:00', 'sent 1 bytes received 2 bytes', 'total size is 3']:
assert not bar.update(record)
bar.clear()
assert capsys.readouterr().err == ''
def test_narrow_terminal(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((20, 24)))
assert sb._ProgressBar().update(' 6.00M 100% 1.48MB/s 0:00:03 (xfr#3, to-chk=0/4)')
# No room for the bar
assert capsys.readouterr().err == '\r100% 6.00M 1.48MB\033[K'
class FakePopen: class FakePopen:
"""Stands in for rsync: creates the destination and returns a chosen return code""" """Stands in for rsync: creates the destination and returns a chosen return code"""
@@ -195,6 +276,8 @@ class FakePopen:
def __init__(self, args: list[str], **kwargs: Any) -> None: def __init__(self, args: list[str], **kwargs: Any) -> None:
FakePopen.calls.append(args) FakePopen.calls.append(args)
os.makedirs(args[args.index('/') + 1]) os.makedirs(args[args.index('/') + 1])
self.stdin = io.BytesIO()
self.stdout = io.BytesIO(b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n')
def __enter__(self) -> 'FakePopen': def __enter__(self) -> 'FakePopen':
return self return self
@@ -202,8 +285,8 @@ class FakePopen:
def __exit__(self, *args: Any) -> None: def __exit__(self, *args: Any) -> None:
pass pass
def communicate(self) -> tuple[bytes, None]: def wait(self) -> int:
return b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n', None return FakePopen.returncode
@pytest.fixture @pytest.fixture
+4 -2
View File
@@ -257,6 +257,8 @@ class RecordPopen:
def __init__(self, args: list[str], **kwargs: Any) -> None: def __init__(self, args: list[str], **kwargs: Any) -> None:
RecordPopen.calls.append(args) RecordPopen.calls.append(args)
self.stdin = io.BytesIO()
self.stdout = io.BytesIO(b'sent 1 bytes received 2 bytes\ntotal size is 3\n')
def __enter__(self) -> 'RecordPopen': def __enter__(self) -> 'RecordPopen':
return self return self
@@ -264,8 +266,8 @@ class RecordPopen:
def __exit__(self, *args: Any) -> None: def __exit__(self, *args: Any) -> None:
pass pass
def communicate(self) -> tuple[bytes, None]: def wait(self) -> int:
return b'sent 1 bytes received 2 bytes\ntotal size is 3\n', None return RecordPopen.returncode
@pytest.mark.usefixtures('connect') @pytest.mark.usefixtures('connect')