Compare commits

..
9 Commits
Author SHA1 Message Date
Fuxino a7fd325fa7 Bump version to 4.4.1 2026-10-15 20:34:13 +02:00
Fuxino 8a479a8455 Base the progress bar on file counts
With --link-dest unchanged files are linked, not copied, so rsync's byte
percentage stayed near zero and the ETA was hours. Use the checked/total
file counts instead, and show no percentage or ETA while still scanning.
2026-10-15 20:32:44 +02:00
Fuxino 3e486eb3ab Install README in PKGBUILD
Install README.md to /usr/share/doc, bump pkgrel to 2 and quote variables.
2026-10-14 13:25:41 +02:00
Fuxino 331e72b373 Update PKGBUILD
Update to 4.4.0 and run the test suite in check().
2026-10-14 09:49:46 +02:00
Fuxino 13cae98776 Bump version to 4.4.0 2026-10-14 09:48:18 +02:00
Fuxino 7cd90f8d11 Add a progress bar for rsync
On a terminal, run rsync with --info=progress2 and draw its updates as a
progress bar on stderr. The ETA is estimated locally, since rsync reports
the elapsed time when a file finishes. Disable it with --no-progress.
2026-10-14 09:46:50 +02:00
Fuxino d0fb5b65fb Read rsync output while it runs
Read the output as it arrives, split on newlines and carriage returns,
instead of all at once with communicate(). This is needed for the
progress bar. Invalid UTF-8 in the output is replaced instead of raising.
2026-10-14 09:42:37 +02:00
Fuxino a3ace84c8b Log each old backup as it is removed
Removing old backups can take minutes on a slow drive; logging each one with
its position (e.g. "(1/3)") shows that the backup is still progressing.
2026-10-14 09:40:35 +02:00
Fuxino 10eccd360c Add mypy configuration
Check both the package and the tests by default, require type hints on all functions and ignore the missing stubs for paramiko, systemd and dbus.
2026-10-14 08:39:46 +02:00
8 changed files with 275 additions and 31 deletions

No files matched your search

+20 -10
View File
@@ -4,9 +4,9 @@
pkgname=simple_backup
pkgdesc='Simple backup script that uses rsync to copy files'
pkgver=4.3.0
pkgrel=1
url="https://git.shouldnt.work/fuxino/${pkgname}"
pkgver=4.4.0
pkgrel=2
url="https://git.shouldnt.work/fuxino/$pkgname"
arch=('any')
license=('GPL-3.0-or-later')
makedepends=('git'
@@ -18,21 +18,31 @@ depends=('python>=3.10'
optdepends=('python-systemd: use systemd log'
'python-dbus: for desktop notifications'
'python-paramiko: for remote backup through ssh')
checkdepends=('python-pytest'
'python-paramiko'
'openssh')
conflicts=('simple_backup-git')
source=(git+${url}?signed#tag=${pkgver})
source=("git+$url?signed#tag=$pkgver")
validpgpkeys=('7E12BC1FF3B6EDB2CD8053EB981A2B2A3BBF5514')
sha256sums=('7d9ea50d5829ac3c681a979745f1ae7afd2453eb46ceeae4f441ef5ae4540bd2')
sha256sums=('cc5944558ed4bc30a0d1a00b49dfced371909dd6c4f39b0c1ccf8f39b445e907')
build()
{
cd ${srcdir}/${pkgname}
cd "$srcdir"/"$pkgname"
python -m build --wheel --no-isolation
}
check()
{
cd "$srcdir"/"$pkgname"
python -m pytest
}
package()
{
cd ${srcdir}/${pkgname}
python -m installer --destdir=${pkgdir} dist/*.whl
install -Dm644 ${pkgname}/${pkgname}.conf ${pkgdir}/usr/share/doc/${pkgname}/${pkgname}.conf
install -Dm644 man/${pkgname}.1 ${pkgdir}/usr/share/man/man1/${pkgname}.1
cd "$srcdir"/"$pkgname"
python -m installer --destdir="$pkgdir" dist/*.whl
install -Dm644 "$pkgname"/"$pkgname".conf "$pkgdir"/usr/share/doc/"$pkgname"/"$pkgname".conf
install -Dm644 README.md "$pkgdir"/usr/share/doc/"$pkgname"/README.md
install -Dm644 man/"$pkgname".1 "$pkgdir"/usr/share/man/man1/"$pkgname".1
}
+7 -1
View File
@@ -1,4 +1,4 @@
.TH SIMPLE_BACKUP 1 2026-10-08 SIMPLE_BACKUP 4.3.0
.TH SIMPLE_BACKUP 1 2026-10-15 SIMPLE_BACKUP 4.4.1
.SH NAME
simple_backup \- Backup files and folders using rsync
.SH SYNOPSIS
@@ -105,6 +105,12 @@ according to \-\-keep will be left, and the new backup may be incomplete. In par
.B \-\-no\-syslog
Don't use systemd journal for logging.
.TP
.B \-\-no\-progress
Don't show a progress bar while rsync copies the files. The progress bar is only shown when
the output is a terminal, so it never appears in the systemd journal or in redirected output.
While rsync is still looking for files, the bar shows \(aq(scanning)\(aq, and the percentage
and estimated time are only approximate.
.TP
.B \-\-rsync\-options OPTIONS [OPTION...]
By default, the following rsync options are used:
.RS
+10
View File
@@ -47,6 +47,16 @@ packages = ["simple_backup"]
[tool.setuptools.dynamic]
version = { attr = "simple_backup.__version__" }
[tool.mypy]
files = ["simple_backup", "tests"]
disallow_untyped_defs = true
warn_unused_ignores = true
# No usable stubs (types-paramiko lags behind paramiko 5)
[[tool.mypy.overrides]]
module = ["paramiko", "systemd", "dbus"]
ignore_missing_imports = true
[tool.pytest.ini_options]
testpaths = ["tests"]
# Import the repo copy, not an installed one
+1 -1
View File
@@ -1,3 +1,3 @@
"""Init."""
__version__ = '4.3.0'
__version__ = '4.4.1'
+122 -12
View File
@@ -18,10 +18,12 @@ from __future__ import annotations
import sys
import os
import pwd
from typing import Callable, List, Optional, ParamSpec, TypedDict, TypeVar, Union
import io
import re
from typing import Callable, Iterator, List, Optional, ParamSpec, TypedDict, TypeVar
import warnings
from functools import wraps
from shutil import rmtree, which
from shutil import get_terminal_size, rmtree, which
import shlex
import argparse
import configparser
@@ -72,6 +74,10 @@ R = TypeVar('R')
# Seconds to wait for the TCP connection to the SSH server
SSH_TIMEOUT = 15
# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
PROGRESS_RE = re.compile(r'\s*(\S+)\s+\d+%\s+\S+\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=(\d+)/(\d+)\))?\s*')
PROGRESS_BAR_WIDTH = 30
def timing(func: Callable[P, R]) -> Callable[P, R]:
"""Decorator to measure execution time of a function
@@ -98,6 +104,66 @@ class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFo
"""Custom format for argparse help text"""
class _ProgressBar:
"""Draws rsync --info=progress2 updates as a bar on stderr"""
def __init__(self) -> None:
self._start = default_timer()
# rsync reports ir-chk while incremental recursion is still finding files
self._scanning = True
self._checked = 0
self._total = 0
self._shown = False
def update(self, record: str) -> bool:
"""Draw the bar if record is a progress update, and return whether it was"""
m = PROGRESS_RE.fullmatch(record)
if m is None:
return False
size, chk = m.group(1), m.group(2)
# Records in the middle of a file have no counts, so keep the last ones
if chk is not None:
self._scanning = chk == 'ir'
self._total = int(m.group(4))
self._checked = self._total - int(m.group(3))
# Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out
if self._scanning or self._total == 0:
text = f'{self._checked:,} files checked {size} copied (scanning)'
percent = None
else:
percent = self._checked * 100 // self._total
text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied'
if 0 < percent < 100:
eta = int((default_timer() - self._start) * (self._total - self._checked) / self._checked)
text += f' ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
width = get_terminal_size().columns - 1
# Fixed width, so the bar doesn't jump when the text changes length; no bar until the total is known
if percent is not None and len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
filled = PROGRESS_BAR_WIDTH * percent // 100
text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}] {text}'
sys.stderr.write(f'\r{text[:width]}\033[K')
sys.stderr.flush()
self._shown = True
return True
def clear(self) -> None:
"""Remove the bar, so that log messages start on a clean line"""
if self._shown:
sys.stderr.write('\r\033[K')
sys.stderr.flush()
class Backup:
"""Main class defining parameters and functions for performing backup
@@ -122,6 +188,8 @@ class Backup:
Run remote rsync with sudo
remove_before: bool
Indicate if removing old backups will be performed before copying files
progress: bool
Show a progress bar while rsync runs
Methods:
check_params():
@@ -139,7 +207,8 @@ class Backup:
def __init__(self, inputs: Optional[List[str]], output: Optional[str], exclude: Optional[List[str]],
keep: Optional[int], options: str,
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None,
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False) -> None:
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False,
progress: bool = False) -> None:
# Missing values are reported by check_params()
self.inputs = inputs if inputs is not None else []
self.output = output if output is not None else ''
@@ -152,6 +221,7 @@ class Backup:
self.remote_sudo = remote_sudo
self._remove_before = remove_before
self._verbose = verbose
self._progress = progress
self._last_backup = ''
self._server = ''
self._output_dir = ''
@@ -278,7 +348,11 @@ class Backup:
logger.info('Removing old backups...')
dirs.sort()
for i in range(n_backup - self.keep):
n_remove = n_backup - self.keep
for i in range(n_remove):
logger.info('Removing backup %s (%d/%d)...', dirs[i], i + 1, n_remove)
if self.remote_sudo:
_, _, stderr = self._ssh.exec_command(
f'sudo rm -r {shlex.quote(f"{self.output}/simple_backup/{dirs[i]}")}')
@@ -317,7 +391,11 @@ class Backup:
logger.info('Removing old backups...')
dirs.sort()
for i in range(n_backup - self.keep):
n_remove = n_backup - self.keep
for i in range(n_remove):
logger.info('Removing backup %s (%d/%d)...', dirs[i], i + 1, n_remove)
try:
rmtree(f'{self.output}/simple_backup/{dirs[i]}')
count += 1
@@ -638,25 +716,39 @@ class Backup:
if self._remote and self.remote_sudo:
args.append('--rsync-path=sudo rsync')
bar = None
if self._progress:
args.append('--info=progress2')
bar = _ProgressBar()
with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p:
output: Union[bytes, List[str]]
output, _ = p.communicate()
assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
p.stdin.close()
output = []
try:
for r in _read_records(p.stdout):
if r != '' and (bar is None or not bar.update(r)):
output.append(r)
finally:
if bar is not None:
bar.clear()
try:
del os.environ['SSHPASS']
except KeyError:
pass
returncode = p.returncode
output = output.decode("utf-8").split('\n')
returncode = p.wait()
if returncode == 0:
if self._verbose:
logger.info('rsync: %s', output)
else:
logger.info('rsync: %s', output[-3])
logger.info('rsync: %s', output[-2])
logger.info('rsync: %s', output[-1])
else:
self._returncode_log(returncode)
@@ -701,6 +793,21 @@ class Backup:
return 0
def _read_records(stream: io.BufferedIOBase) -> Iterator[str]:
"""Yield rsync output as it arrives, split on '\\n' and '\\r' (which ends progress updates)"""
buffer = b''
while chunk := stream.read1(65536):
*records, buffer = re.split(rb'[\r\n]', buffer + chunk)
for r in records:
yield r.decode('utf-8', errors='replace')
if buffer != b'':
yield buffer.decode('utf-8', errors='replace')
def _detect_user() -> Optional[str]:
if os.geteuid() == 0:
return os.getenv('SUDO_USER') or os.getenv('DOAS_USER')
@@ -741,6 +848,8 @@ def _parse_arguments() -> argparse.Namespace:
help='Remove old backups before executing the backup, instead of after')
parser.add_argument('--no-syslog', action='store_true',
help='Disable systemd journal logging')
parser.add_argument('--no-progress', action='store_true',
help='Don\'t show a progress bar (it is only shown on a terminal)')
parser.add_argument('--rsync-options', nargs='+',
choices=['a', 'l', 'p', 't', 'g', 'o',
'c', 'h', 's', 'D', 'H', 'X'],
@@ -1014,7 +1123,8 @@ def simple_backup() -> int:
rsync_options.append('--numeric-ids')
backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile,
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose)
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose,
progress=not args.no_progress and sys.stderr.isatty())
try:
return_code = backup.check_params(homedir)
+14 -1
View File
@@ -27,7 +27,7 @@ class FakeBackup:
def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str,
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None,
ssh_keyfile: Optional[str] = None, remote_sudo: bool = False,
remove_before: bool = False, verbose: bool = False) -> None:
remove_before: bool = False, verbose: bool = False, progress: bool = False) -> None:
self.inputs = inputs
self.output = output
self.exclude = exclude
@@ -39,6 +39,7 @@ class FakeBackup:
self.remote_sudo = remote_sudo
self.remove_before = remove_before
self.verbose = verbose
self.progress = progress
self.homedir: Optional[str] = None
self.ran = False
self.closed = False
@@ -224,6 +225,18 @@ ssh_keyfile=/keys/id
b = last_backup()
assert b.remote_sudo and b.remove_before and b.verbose
@pytest.mark.parametrize('tty, argv, progress', [
(True, [], True),
(True, ['--no-progress'], False),
(False, [], False),
])
def test_progress(self, run_cli: RunCli, config: Callable[[str], str], monkeypatch: pytest.MonkeyPatch,
tty: bool, argv: list[str], progress: bool) -> None:
monkeypatch.setattr(sb.sys.stderr, 'isatty', lambda: tty)
assert run_cli('-c', config('backup_dir=/backups'), *argv) == 0
assert last_backup().progress is progress
def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None:
FakeBackup.check_code = 2
+97 -4
View File
@@ -1,5 +1,6 @@
"""Tests for local backups, with real rsync runs and with a fake Popen"""
import io
import os
from datetime import datetime, timedelta
from pathlib import Path
@@ -52,8 +53,9 @@ def out(tmp_path: Path) -> Path:
def make_backup(out: Path, inputs: list[str], keep: int = -1, exclude: Optional[list[str]] = None,
remove_before: bool = False) -> sb.Backup:
backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before)
remove_before: bool = False, progress: bool = False) -> sb.Backup:
backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before,
progress=progress)
assert backup.check_params() == 0
@@ -140,6 +142,15 @@ class TestRsync:
assert backups(out) == remaining
def test_removal_logged(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture) -> None:
for _ in range(3):
assert run_backup(out, [str(src)]) == 0
assert run_backup(out, [str(src)], keep=0) == 0
assert 'Removing backup 2026-10-14_10-00-00 (1/3)...' in caplog.text
assert 'Removing backup 2026-10-14_12-00-00 (3/3)...' in caplog.text
assert 'Removed 3 backups' in caplog.text
def test_remove_before(self, src: Path, out: Path) -> None:
assert run_backup(out, [str(src)]) == 0
assert run_backup(out, [str(src)], keep=0, remove_before=True) == 0
@@ -177,6 +188,16 @@ class TestRsync:
assert run_backup(out, [str(src)], keep=0) == 0
assert backups(out) == ['2026-10-14_12-00-00']
def test_progress(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture,
capsys: pytest.CaptureFixture[str]) -> None:
assert run_backup(out, [str(src)], progress=True) == 0
assert '100%' in capsys.readouterr().err
# Only the summary is logged, not the progress updates
assert 'rsync: sent' in caplog.text
assert 'rsync: total size' in caplog.text
assert '%' not in caplog.text
def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None:
config_file = str(tmp_path / 'missing.conf')
@@ -186,6 +207,76 @@ class TestRsync:
assert backups(out) == ['2026-10-14_11-00-00']
class ChunkStream(io.BufferedIOBase):
"""Returns the given chunks one per read1() call, like a pipe"""
def __init__(self, chunks: list[bytes]) -> None:
self.chunks = chunks
def read1(self, size: int = -1) -> bytes:
return self.chunks.pop(0) if self.chunks else b''
class TestReadRecords:
def test_split_on_newline_and_cr(self) -> None:
stream = ChunkStream([b'file\n\r 1.00M 50%\r 2.00M 100%\n\nsent 1 bytes\ntotal'])
assert list(sb._read_records(stream)) == ['file', '', ' 1.00M 50%', ' 2.00M 100%', '', 'sent 1 bytes',
'total']
def test_records_across_chunks(self) -> None:
stream = ChunkStream([b'fi', b'le\r', b'\nna', 'mé\n'.encode(), b'\xff\n'])
assert list(sb._read_records(stream)) == ['file', '', 'namé', '�']
class TestProgressBar:
@pytest.fixture(autouse=True)
def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24)))
# 30 s elapsed at each update
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
def test_scanning(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=1009/2632)')
assert capsys.readouterr().err == '\r1,623 files checked 2.00M copied (scanning)\033[K'
def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
# With --link-dest rsync's percentage leaves out linked files, so the files are counted instead
assert bar.update(' 4,000,000 1% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)')
assert capsys.readouterr().err == \
'\r[######################--------] 75% 3/4 files 4,000,000 copied ETA 0:00:10\033[K'
def test_keeps_counts_within_file(self, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0, 30.0]).__next__)
bar = sb._ProgressBar()
assert bar.update(' 2.00M 1% 1.47MB/s 0:00:02 (xfr#1, to-chk=2/4)')
assert bar.update(' 3.00M 2% 1.47MB/s 0:00:03')
assert capsys.readouterr().err.endswith('] 50% 2/4 files 3.00M copied ETA 0:00:30\033[K')
def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None:
bar = sb._ProgressBar()
for record in ['home/user/file 50% 1:00:00', 'sent 1 bytes received 2 bytes', 'total size is 3']:
assert not bar.update(record)
bar.clear()
assert capsys.readouterr().err == ''
def test_narrow_terminal(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((20, 24)))
assert sb._ProgressBar().update(' 6.00M 100% 1.48MB/s 0:00:03 (xfr#3, to-chk=0/4)')
# No room for the bar
assert capsys.readouterr().err == '\r100% 4/4 files 6.\033[K'
class FakePopen:
"""Stands in for rsync: creates the destination and returns a chosen return code"""
@@ -195,6 +286,8 @@ class FakePopen:
def __init__(self, args: list[str], **kwargs: Any) -> None:
FakePopen.calls.append(args)
os.makedirs(args[args.index('/') + 1])
self.stdin = io.BytesIO()
self.stdout = io.BytesIO(b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n')
def __enter__(self) -> 'FakePopen':
return self
@@ -202,8 +295,8 @@ class FakePopen:
def __exit__(self, *args: Any) -> None:
pass
def communicate(self) -> tuple[bytes, None]:
return b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n', None
def wait(self) -> int:
return FakePopen.returncode
@pytest.fixture
+4 -2
View File
@@ -257,6 +257,8 @@ class RecordPopen:
def __init__(self, args: list[str], **kwargs: Any) -> None:
RecordPopen.calls.append(args)
self.stdin = io.BytesIO()
self.stdout = io.BytesIO(b'sent 1 bytes received 2 bytes\ntotal size is 3\n')
def __enter__(self) -> 'RecordPopen':
return self
@@ -264,8 +266,8 @@ class RecordPopen:
def __exit__(self, *args: Any) -> None:
pass
def communicate(self) -> tuple[bytes, None]:
return b'sent 1 bytes received 2 bytes\ntotal size is 3\n', None
def wait(self) -> int:
return RecordPopen.returncode
@pytest.mark.usefixtures('connect')