Read rsync output while it runs

Read the output as it arrives, split on newlines and carriage returns,
instead of all at once with communicate(). This is needed for the
progress bar. Invalid UTF-8 in the output is replaced instead of raising.

simple_backup/simple_backup.py

@@ -18,7 +18,9 @@ from __future__ import annotations
 import sys
 import os
 import pwd
-from typing import Callable, List, Optional, ParamSpec, TypedDict, TypeVar, Union
+import io
+import re
+from typing import Callable, Iterator, List, Optional, ParamSpec, TypedDict, TypeVar
 import warnings
 from functools import wraps
 from shutil import rmtree, which
@@ -647,24 +649,24 @@ class Backup:
             args.append('--rsync-path=sudo rsync')
 
         with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p:
-            output: Union[bytes, List[str]]
-            output, _ = p.communicate()
+            assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
+            p.stdin.close()
+
+            output = [r for r in _read_records(p.stdout) if r != '']
 
             try:
                 del os.environ['SSHPASS']
             except KeyError:
                 pass
 
-            returncode = p.returncode
-
-        output = output.decode("utf-8").split('\n')
+            returncode = p.wait()
 
         if returncode == 0:
             if self._verbose:
                 logger.info('rsync: %s', output)
             else:
-                logger.info('rsync: %s', output[-3])
                 logger.info('rsync: %s', output[-2])
+                logger.info('rsync: %s', output[-1])
         else:
             self._returncode_log(returncode)
 
@@ -709,6 +711,21 @@ class Backup:
         return 0
 
 
+def _read_records(stream: io.BufferedIOBase) -> Iterator[str]:
+    """Yield rsync output as it arrives, split on '\\n' and '\\r' (which ends progress updates)"""
+
+    buffer = b''
+
+    while chunk := stream.read1(65536):
+        *records, buffer = re.split(rb'[\r\n]', buffer + chunk)
+
+        for r in records:
+            yield r.decode('utf-8', errors='replace')
+
+    if buffer != b'':
+        yield buffer.decode('utf-8', errors='replace')
+
+
 def _detect_user() -> Optional[str]:
     if os.geteuid() == 0:
         return os.getenv('SUDO_USER') or os.getenv('DOAS_USER')