Add a progress bar for rsync
On a terminal, run rsync with --info=progress2 and draw its updates as a progress bar on stderr. The ETA is estimated locally, since rsync reports the elapsed time when a file finishes. Disable it with --no-progress.
This commit is contained in:
Fuxino
@@ -105,6 +105,12 @@ according to \-\-keep will be left, and the new backup may be incomplete. In par
|
||||
.B \-\-no\-syslog
|
||||
Don't use systemd journal for logging.
|
||||
.TP
|
||||
.B \-\-no\-progress
|
||||
Don't show a progress bar while rsync copies the files. The progress bar is only shown when
|
||||
the output is a terminal, so it never appears in the systemd journal or in redirected output.
|
||||
While rsync is still looking for files, the bar shows \(aq(scanning)\(aq, and the percentage
|
||||
and estimated time are only approximate.
|
||||
.TP
|
||||
.B \-\-rsync\-options OPTIONS [OPTION...]
|
||||
By default, the following rsync options are used:
|
||||
.RS
|
||||
|
||||
@@ -23,7 +23,7 @@ import re
|
||||
from typing import Callable, Iterator, List, Optional, ParamSpec, TypedDict, TypeVar
|
||||
import warnings
|
||||
from functools import wraps
|
||||
from shutil import rmtree, which
|
||||
from shutil import get_terminal_size, rmtree, which
|
||||
import shlex
|
||||
import argparse
|
||||
import configparser
|
||||
@@ -74,6 +74,10 @@ R = TypeVar('R')
|
||||
# Seconds to wait for the TCP connection to the SSH server
|
||||
SSH_TIMEOUT = 15
|
||||
|
||||
# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
|
||||
PROGRESS_RE = re.compile(r'\s*(\S+)\s+(\d+)%\s+(\S+)\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=\d+/\d+\))?\s*')
|
||||
PROGRESS_BAR_WIDTH = 30
|
||||
|
||||
|
||||
def timing(func: Callable[P, R]) -> Callable[P, R]:
|
||||
"""Decorator to measure execution time of a function
|
||||
@@ -100,6 +104,59 @@ class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFo
|
||||
"""Custom format for argparse help text"""
|
||||
|
||||
|
||||
class _ProgressBar:
|
||||
"""Draws rsync --info=progress2 updates as a bar on stderr"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._start = default_timer()
|
||||
# rsync reports ir-chk while incremental recursion is still finding files
|
||||
self._scanning = True
|
||||
self._shown = False
|
||||
|
||||
def update(self, record: str) -> bool:
|
||||
"""Draw the bar if record is a progress update, and return whether it was"""
|
||||
|
||||
m = PROGRESS_RE.fullmatch(record)
|
||||
|
||||
if m is None:
|
||||
return False
|
||||
|
||||
size, percent, speed, chk = m.group(1), int(m.group(2)), m.group(3), m.group(4)
|
||||
|
||||
if chk is not None:
|
||||
self._scanning = chk == 'ir'
|
||||
|
||||
# rsync's own time field is the elapsed time on records ending a file, so estimate the ETA here
|
||||
if 0 < percent < 100:
|
||||
eta = int((default_timer() - self._start) * (100 - percent) / percent)
|
||||
text = f'{percent:3d}% {size} {speed} ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
|
||||
else:
|
||||
text = f'{percent:3d}% {size} {speed}'
|
||||
|
||||
if self._scanning:
|
||||
text += ' (scanning)'
|
||||
|
||||
width = get_terminal_size().columns - 1
|
||||
|
||||
# Fixed width, so the bar doesn't jump when the text changes length
|
||||
if len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
|
||||
filled = PROGRESS_BAR_WIDTH * percent // 100
|
||||
text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}] {text}'
|
||||
|
||||
sys.stderr.write(f'\r{text[:width]}\033[K')
|
||||
sys.stderr.flush()
|
||||
self._shown = True
|
||||
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Remove the bar, so that log messages start on a clean line"""
|
||||
|
||||
if self._shown:
|
||||
sys.stderr.write('\r\033[K')
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
class Backup:
|
||||
"""Main class defining parameters and functions for performing backup
|
||||
|
||||
@@ -124,6 +181,8 @@ class Backup:
|
||||
Run remote rsync with sudo
|
||||
remove_before: bool
|
||||
Indicate if removing old backups will be performed before copying files
|
||||
progress: bool
|
||||
Show a progress bar while rsync runs
|
||||
|
||||
Methods:
|
||||
check_params():
|
||||
@@ -141,7 +200,8 @@ class Backup:
|
||||
def __init__(self, inputs: Optional[List[str]], output: Optional[str], exclude: Optional[List[str]],
|
||||
keep: Optional[int], options: str,
|
||||
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None, ssh_keyfile: Optional[str] = None,
|
||||
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False) -> None:
|
||||
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False,
|
||||
progress: bool = False) -> None:
|
||||
# Missing values are reported by check_params()
|
||||
self.inputs = inputs if inputs is not None else []
|
||||
self.output = output if output is not None else ''
|
||||
@@ -154,6 +214,7 @@ class Backup:
|
||||
self.remote_sudo = remote_sudo
|
||||
self._remove_before = remove_before
|
||||
self._verbose = verbose
|
||||
self._progress = progress
|
||||
self._last_backup = ''
|
||||
self._server = ''
|
||||
self._output_dir = ''
|
||||
@@ -648,11 +709,25 @@ class Backup:
|
||||
if self._remote and self.remote_sudo:
|
||||
args.append('--rsync-path=sudo rsync')
|
||||
|
||||
bar = None
|
||||
|
||||
if self._progress:
|
||||
args.append('--info=progress2')
|
||||
bar = _ProgressBar()
|
||||
|
||||
with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p:
|
||||
assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
|
||||
p.stdin.close()
|
||||
|
||||
output = [r for r in _read_records(p.stdout) if r != '']
|
||||
output = []
|
||||
|
||||
try:
|
||||
for r in _read_records(p.stdout):
|
||||
if r != '' and (bar is None or not bar.update(r)):
|
||||
output.append(r)
|
||||
finally:
|
||||
if bar is not None:
|
||||
bar.clear()
|
||||
|
||||
try:
|
||||
del os.environ['SSHPASS']
|
||||
@@ -766,6 +841,8 @@ def _parse_arguments() -> argparse.Namespace:
|
||||
help='Remove old backups before executing the backup, instead of after')
|
||||
parser.add_argument('--no-syslog', action='store_true',
|
||||
help='Disable systemd journal logging')
|
||||
parser.add_argument('--no-progress', action='store_true',
|
||||
help='Don\'t show a progress bar (it is only shown on a terminal)')
|
||||
parser.add_argument('--rsync-options', nargs='+',
|
||||
choices=['a', 'l', 'p', 't', 'g', 'o',
|
||||
'c', 'h', 's', 'D', 'H', 'X'],
|
||||
@@ -1039,7 +1116,8 @@ def simple_backup() -> int:
|
||||
rsync_options.append('--numeric-ids')
|
||||
|
||||
backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile,
|
||||
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose)
|
||||
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose,
|
||||
progress=not args.no_progress and sys.stderr.isatty())
|
||||
|
||||
try:
|
||||
return_code = backup.check_params(homedir)
|
||||
|
||||
+14
-1
@@ -27,7 +27,7 @@ class FakeBackup:
|
||||
def __init__(self, inputs: Any, output: Any, exclude: Any, keep: Any, options: str,
|
||||
ssh_host: Optional[str] = None, ssh_user: Optional[str] = None,
|
||||
ssh_keyfile: Optional[str] = None, remote_sudo: bool = False,
|
||||
remove_before: bool = False, verbose: bool = False) -> None:
|
||||
remove_before: bool = False, verbose: bool = False, progress: bool = False) -> None:
|
||||
self.inputs = inputs
|
||||
self.output = output
|
||||
self.exclude = exclude
|
||||
@@ -39,6 +39,7 @@ class FakeBackup:
|
||||
self.remote_sudo = remote_sudo
|
||||
self.remove_before = remove_before
|
||||
self.verbose = verbose
|
||||
self.progress = progress
|
||||
self.homedir: Optional[str] = None
|
||||
self.ran = False
|
||||
self.closed = False
|
||||
@@ -224,6 +225,18 @@ ssh_keyfile=/keys/id
|
||||
b = last_backup()
|
||||
assert b.remote_sudo and b.remove_before and b.verbose
|
||||
|
||||
@pytest.mark.parametrize('tty, argv, progress', [
|
||||
(True, [], True),
|
||||
(True, ['--no-progress'], False),
|
||||
(False, [], False),
|
||||
])
|
||||
def test_progress(self, run_cli: RunCli, config: Callable[[str], str], monkeypatch: pytest.MonkeyPatch,
|
||||
tty: bool, argv: list[str], progress: bool) -> None:
|
||||
monkeypatch.setattr(sb.sys.stderr, 'isatty', lambda: tty)
|
||||
|
||||
assert run_cli('-c', config('backup_dir=/backups'), *argv) == 0
|
||||
assert last_backup().progress is progress
|
||||
|
||||
def test_check_params_failure(self, run_cli: RunCli, config: Callable[[str], str]) -> None:
|
||||
FakeBackup.check_code = 2
|
||||
|
||||
|
||||
+50
-2
@@ -53,8 +53,9 @@ def out(tmp_path: Path) -> Path:
|
||||
|
||||
|
||||
def make_backup(out: Path, inputs: list[str], keep: int = -1, exclude: Optional[list[str]] = None,
|
||||
remove_before: bool = False) -> sb.Backup:
|
||||
backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before)
|
||||
remove_before: bool = False, progress: bool = False) -> sb.Backup:
|
||||
backup = sb.Backup(inputs, str(out), exclude or [], keep, OPTIONS, remove_before=remove_before,
|
||||
progress=progress)
|
||||
|
||||
assert backup.check_params() == 0
|
||||
|
||||
@@ -187,6 +188,16 @@ class TestRsync:
|
||||
assert run_backup(out, [str(src)], keep=0) == 0
|
||||
assert backups(out) == ['2026-10-14_12-00-00']
|
||||
|
||||
def test_progress(self, src: Path, out: Path, caplog: pytest.LogCaptureFixture,
|
||||
capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert run_backup(out, [str(src)], progress=True) == 0
|
||||
|
||||
assert '100%' in capsys.readouterr().err
|
||||
# Only the summary is logged, not the progress updates
|
||||
assert 'rsync: sent' in caplog.text
|
||||
assert 'rsync: total size' in caplog.text
|
||||
assert '%' not in caplog.text
|
||||
|
||||
def test_cli(self, src: Path, out: Path, tmp_path: Path, run_cli: Callable[..., int]) -> None:
|
||||
config_file = str(tmp_path / 'missing.conf')
|
||||
|
||||
@@ -219,6 +230,43 @@ class TestReadRecords:
|
||||
assert list(sb._read_records(stream)) == ['file', '', 'namé', '�']
|
||||
|
||||
|
||||
class TestProgressBar:
|
||||
@pytest.fixture(autouse=True)
|
||||
def terminal(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((81, 24)))
|
||||
# 30 s elapsed at each update
|
||||
monkeypatch.setattr(sb, 'default_timer', iter([0.0, 30.0]).__next__)
|
||||
|
||||
def test_update(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
bar = sb._ProgressBar()
|
||||
|
||||
assert bar.update(' 2.00M 25% 1.47MB/s 0:00:02 (xfr#1, ir-chk=2/4)')
|
||||
assert capsys.readouterr().err == \
|
||||
'\r[#######-----------------------] 25% 2.00M 1.47MB/s ETA 0:01:30 (scanning)\033[K'
|
||||
|
||||
def test_without_human_readable(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
bar = sb._ProgressBar()
|
||||
|
||||
assert bar.update(' 4,000,000 50% 2.97MB/s 0:00:01 (xfr#2, to-chk=1/4)')
|
||||
assert capsys.readouterr().err.endswith('] 50% 4,000,000 2.97MB/s ETA 0:00:30\033[K')
|
||||
|
||||
def test_not_progress(self, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
bar = sb._ProgressBar()
|
||||
|
||||
for record in ['home/user/file 50% 1:00:00', 'sent 1 bytes received 2 bytes', 'total size is 3']:
|
||||
assert not bar.update(record)
|
||||
|
||||
bar.clear()
|
||||
assert capsys.readouterr().err == ''
|
||||
|
||||
def test_narrow_terminal(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
monkeypatch.setattr(sb, 'get_terminal_size', lambda: os.terminal_size((20, 24)))
|
||||
|
||||
assert sb._ProgressBar().update(' 6.00M 100% 1.48MB/s 0:00:03 (xfr#3, to-chk=0/4)')
|
||||
# No room for the bar
|
||||
assert capsys.readouterr().err == '\r100% 6.00M 1.48MB\033[K'
|
||||
|
||||
|
||||
class FakePopen:
|
||||
"""Stands in for rsync: creates the destination and returns a chosen return code"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user