Read rsync output while it runs

Read the output as it arrives, split on newlines and carriage returns,
instead of all at once with communicate(). This is needed for the
progress bar. Invalid UTF-8 in the output is replaced instead of raising.
This commit is contained in:
Fuxino
2026-10-14 09:42:37 +02:00
parent a3ace84c8b
commit d0fb5b65fb
3 changed files with 56 additions and 11 deletions
+28 -2
View File
@@ -1,5 +1,6 @@
"""Tests for local backups, with real rsync runs and with a fake Popen"""
import io
import os
from datetime import datetime, timedelta
from pathlib import Path
@@ -195,6 +196,29 @@ class TestRsync:
assert backups(out) == ['2026-10-14_11-00-00']
class ChunkStream(io.BufferedIOBase):
"""Returns the given chunks one per read1() call, like a pipe"""
def __init__(self, chunks: list[bytes]) -> None:
self.chunks = chunks
def read1(self, size: int = -1) -> bytes:
return self.chunks.pop(0) if self.chunks else b''
class TestReadRecords:
def test_split_on_newline_and_cr(self) -> None:
stream = ChunkStream([b'file\n\r 1.00M 50%\r 2.00M 100%\n\nsent 1 bytes\ntotal'])
assert list(sb._read_records(stream)) == ['file', '', ' 1.00M 50%', ' 2.00M 100%', '', 'sent 1 bytes',
'total']
def test_records_across_chunks(self) -> None:
stream = ChunkStream([b'fi', b'le\r', b'\nna', 'mé\n'.encode(), b'\xff\n'])
assert list(sb._read_records(stream)) == ['file', '', 'namé', '�']
class FakePopen:
"""Stands in for rsync: creates the destination and returns a chosen return code"""
@@ -204,6 +228,8 @@ class FakePopen:
def __init__(self, args: list[str], **kwargs: Any) -> None:
FakePopen.calls.append(args)
os.makedirs(args[args.index('/') + 1])
self.stdin = io.BytesIO()
self.stdout = io.BytesIO(b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n')
def __enter__(self) -> 'FakePopen':
return self
@@ -211,8 +237,8 @@ class FakePopen:
def __exit__(self, *args: Any) -> None:
pass
def communicate(self) -> tuple[bytes, None]:
return b'sending incremental file list\nsent 1 bytes received 2 bytes\ntotal size is 3\n', None
def wait(self) -> int:
return FakePopen.returncode
@pytest.fixture