Fuxino
6d54019a18
Write the completion time and backup directory to $XDG_STATE_HOME/simple_backup/last_backup after each successful backup (as the target user when run as root), for status bar reminders. --last-backup prints it.
1293 lines
43 KiB
Python
Executable File
1293 lines
43 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""
|
|
A simple python script that calls rsync to perform a backup
|
|
|
|
Parameters can be specified on the command line or using a configuration file
|
|
Backup to a remote server is also supported (experimental)
|
|
|
|
Classes:
|
|
MyFormatter
|
|
Backup
|
|
"""
|
|
|
|
# Avoid evaluating annotations at definition time, as they reference optional modules (paramiko)
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import configparser
|
|
import io
|
|
import logging
|
|
import os
|
|
import pwd
|
|
import re
|
|
import shlex
|
|
import sys
|
|
import warnings
|
|
from collections import deque
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import datetime
|
|
from functools import wraps
|
|
from getpass import GetPassWarning, getpass
|
|
from glob import glob
|
|
from logging import StreamHandler
|
|
from shutil import get_terminal_size, rmtree, which
|
|
from subprocess import PIPE, STDOUT, Popen
|
|
from tempfile import mkstemp
|
|
from timeit import default_timer
|
|
from typing import Any, ParamSpec, TypedDict, TypeVar
|
|
|
|
try:
|
|
import paramiko
|
|
except ImportError:
|
|
pass
|
|
|
|
try:
|
|
from systemd import journal
|
|
except ImportError:
|
|
journal = None
|
|
|
|
try:
|
|
import dbus
|
|
except ImportError:
|
|
dbus = None
|
|
|
|
|
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
logger = logging.getLogger(os.path.basename(__file__))
|
|
c_handler = StreamHandler()
|
|
|
|
c_handler.setLevel(logging.INFO)
|
|
c_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
|
|
c_handler.setFormatter(c_format)
|
|
logger.addHandler(c_handler)
|
|
|
|
if journal:
|
|
j_handler = journal.JournalHandler()
|
|
j_handler.setLevel(logging.INFO)
|
|
j_format = logging.Formatter('%(levelname)s - %(message)s')
|
|
j_handler.setFormatter(j_format)
|
|
logger.addHandler(j_handler)
|
|
|
|
P = ParamSpec('P')
|
|
R = TypeVar('R')
|
|
|
|
# Seconds to wait for the TCP connection to the SSH server
|
|
SSH_TIMEOUT = 15
|
|
|
|
# An rsync --info=progress2 update: size, percent, speed, time, then optionally (xfr#N, ir-chk/to-chk=N/N)
|
|
PROGRESS_RE = re.compile(r'\s*(\S+)\s+\d+%\s+\S+\s+\d+:\d\d:\d\d(?:\s+\(xfr#\d+, (ir|to)-chk=(\d+)/(\d+)\))?\s*')
|
|
PROGRESS_BAR_WIDTH = 30
|
|
|
|
|
|
def timing(func: Callable[P, R]) -> Callable[P, R]:
|
|
"""Decorator to measure execution time of a function
|
|
|
|
Parameters:
|
|
func: Function to decorate
|
|
"""
|
|
@wraps(func)
|
|
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
|
start = default_timer()
|
|
|
|
value = func(*args, **kwargs)
|
|
|
|
end = default_timer()
|
|
|
|
logger.info('Elapsed time: %.3f seconds', end - start)
|
|
|
|
return value
|
|
|
|
return wrapper
|
|
|
|
|
|
class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
|
|
"""Custom format for argparse help text"""
|
|
|
|
|
|
class _ProgressBar:
|
|
"""Draws rsync --info=progress2 updates as a bar on stderr"""
|
|
|
|
def __init__(self) -> None:
|
|
self._start = default_timer()
|
|
# rsync reports ir-chk while incremental recursion is still finding files
|
|
self._scanning = True
|
|
self._checked = 0
|
|
self._total = 0
|
|
self._shown = False
|
|
|
|
def update(self, record: str) -> bool:
|
|
"""Draw the bar if record is a progress update, and return whether it was"""
|
|
|
|
m = PROGRESS_RE.fullmatch(record)
|
|
|
|
if m is None:
|
|
return False
|
|
|
|
size, chk = m.group(1), m.group(2)
|
|
|
|
# Records in the middle of a file have no counts, so keep the last ones
|
|
if chk is not None:
|
|
self._scanning = chk == 'ir'
|
|
self._total = int(m.group(4))
|
|
self._checked = self._total - int(m.group(3))
|
|
|
|
# Count files, not bytes: with --link-dest unchanged files are linked, and rsync's percentage leaves them out
|
|
if self._scanning or self._total == 0:
|
|
text = f'{self._checked:,} files checked {size} copied (scanning)'
|
|
percent = None
|
|
else:
|
|
percent = self._checked * 100 // self._total
|
|
text = f'{percent:3d}% {self._checked:,}/{self._total:,} files {size} copied'
|
|
|
|
if 0 < percent < 100:
|
|
eta = int((default_timer() - self._start) * (self._total - self._checked) / self._checked)
|
|
text += f' ETA {eta // 3600}:{eta // 60 % 60:02d}:{eta % 60:02d}'
|
|
|
|
width = get_terminal_size().columns - 1
|
|
|
|
# Fixed width, so the bar doesn't jump when the text changes length; no bar until the total is known
|
|
if percent is not None and len(text) + PROGRESS_BAR_WIDTH + 4 <= width:
|
|
filled = PROGRESS_BAR_WIDTH * percent // 100
|
|
text = f'[{"#" * filled}{"-" * (PROGRESS_BAR_WIDTH - filled)}] {text}'
|
|
|
|
sys.stderr.write(f'\r{text[:width]}\033[K')
|
|
sys.stderr.flush()
|
|
self._shown = True
|
|
|
|
return True
|
|
|
|
def clear(self) -> None:
|
|
"""Remove the bar, so that log messages start on a clean line"""
|
|
|
|
if self._shown:
|
|
sys.stderr.write('\r\033[K')
|
|
sys.stderr.flush()
|
|
|
|
|
|
class Backup:
|
|
"""Main class defining parameters and functions for performing backup
|
|
|
|
Attributes:
|
|
inputs: list
|
|
Files and folders that will be backup up
|
|
output: str
|
|
Path where the backup will be saved
|
|
exclude: list
|
|
List of files/folders/patterns to exclude from backup
|
|
options: str
|
|
String representing main backup options for rsync
|
|
keep: int
|
|
Number of old backup to preserve
|
|
ssh_host: str
|
|
Hostname of server (for remote backup)
|
|
ssh_user: str
|
|
Username for server login (for remote backup)
|
|
ssh_keyfile: str
|
|
Location of ssh key
|
|
remote_sudo: bool
|
|
Run remote rsync with sudo
|
|
remove_before: bool
|
|
Indicate if removing old backups will be performed before copying files
|
|
progress: bool
|
|
Show a progress bar while rsync runs
|
|
|
|
Methods:
|
|
check_params():
|
|
Check if parameters for the backup are valid
|
|
define_backup_dir():
|
|
Define the actual backup dir
|
|
remove_old_backups():
|
|
Remove old backups if there are more than indicated by 'keep'
|
|
find_last_backup():
|
|
Get path of last backup (from last_backup symlink) for rsync --link-dest
|
|
run():
|
|
Perform the backup
|
|
"""
|
|
|
|
def __init__(self, inputs: list[str] | None, output: str | None, exclude: list[str] | None,
|
|
keep: int | None, options: str,
|
|
ssh_host: str | None = None, ssh_user: str | None = None, ssh_keyfile: str | None = None,
|
|
remote_sudo: bool = False, remove_before: bool = False, verbose: bool = False,
|
|
progress: bool = False) -> None:
|
|
# Missing values are reported by check_params()
|
|
self.inputs = inputs if inputs is not None else []
|
|
self.output = output if output is not None else ''
|
|
self.exclude = exclude if exclude is not None else []
|
|
self.options = options
|
|
self.keep = keep if keep is not None else -1
|
|
self.ssh_host = ssh_host
|
|
self.ssh_user = ssh_user
|
|
self.ssh_keyfile = ssh_keyfile
|
|
self.remote_sudo = remote_sudo
|
|
self._remove_before = remove_before
|
|
self._verbose = verbose
|
|
self._progress = progress
|
|
self._last_backup = ''
|
|
self._server = ''
|
|
self._output_dir = ''
|
|
self._inputs_path = ''
|
|
self._exclude_path = ''
|
|
self._remote = False
|
|
self._ssh: paramiko.client.SSHClient | None = None
|
|
# Passed to sshpass through rsync's environment only
|
|
self._password: str | None = None
|
|
self._removed_count = 0
|
|
|
|
def check_params(self, homedir: str = '') -> int:
|
|
"""Check if parameters for the backup are valid"""
|
|
|
|
if len(self.inputs) == 0:
|
|
logger.info(
|
|
'No existing files or directories specified for backup. Nothing to do')
|
|
|
|
return 1
|
|
|
|
if self.output == '':
|
|
logger.critical(
|
|
'No output path specified. Use -o argument or specify output path in configuration file')
|
|
|
|
return 2
|
|
|
|
if self.ssh_host is not None and self.ssh_user is not None:
|
|
self._remote = True
|
|
|
|
if self._remote:
|
|
try:
|
|
self._ssh = self._ssh_connect(homedir)
|
|
except OSError as e:
|
|
# Unknown hostname, connection refused, unreachable host or timeout
|
|
logger.critical('Can\'t connect to the server %s', self.ssh_host)
|
|
logger.critical(e)
|
|
|
|
return 5
|
|
|
|
if self._ssh is None:
|
|
return 5
|
|
|
|
output = self._resolve_remote_path(self.output)
|
|
|
|
if output is None:
|
|
logger.critical('Output path for backup does not exist')
|
|
|
|
return 2
|
|
|
|
self.output = output
|
|
else:
|
|
if not os.path.isdir(self.output):
|
|
logger.critical('Output path for backup does not exist')
|
|
|
|
return 2
|
|
|
|
self.output = os.path.abspath(self.output)
|
|
|
|
return 0
|
|
|
|
def _resolve_remote_path(self, path: str) -> str | None:
|
|
"""Return the absolute path of a directory on the server, or None if it doesn't exist"""
|
|
|
|
assert self._ssh is not None
|
|
|
|
# Leave a leading '~' unquoted, so that the server's shell expands it to the remote home.
|
|
# Relative paths are resolved from the remote home, where the command starts
|
|
if path == '~' or path.startswith('~/'):
|
|
target = f'~/{shlex.quote(path[2:])}'
|
|
else:
|
|
target = shlex.quote(path)
|
|
|
|
_, stdout, _ = self._ssh.exec_command(f'cd -- {target} && pwd')
|
|
|
|
output = stdout.read().decode('utf-8').strip().split('\n')[-1]
|
|
|
|
if not output.startswith('/'):
|
|
return None
|
|
|
|
return output
|
|
|
|
# Function to create the actual backup directory
|
|
def define_backup_dir(self) -> None:
|
|
"""Define the actual backup dir"""
|
|
# Avoid ':', which is not allowed on FAT/exFAT filesystems.
|
|
# Backups created with the old format ('%Y-%m-%d %H:%M:%S') still sort correctly
|
|
now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
|
|
self._output_dir = f'{self.output}/simple_backup/{now}'
|
|
|
|
if self._remote:
|
|
self._server = f'{self.ssh_user}@{self.ssh_host}:'
|
|
|
|
@property
|
|
def destination(self) -> str:
|
|
"""The current backup directory, prefixed with 'user@host:' for remote backups"""
|
|
|
|
return f'{self._server}{self._output_dir}'
|
|
|
|
def remove_old_backups(self) -> None:
|
|
"""Remove old backups if there are more than indicated by 'keep'"""
|
|
|
|
dirs = self._list_backups()
|
|
|
|
if dirs is None:
|
|
return
|
|
|
|
n_backup = len(dirs)
|
|
|
|
# The new backup is already in the list, unless old ones are removed before it
|
|
if not self._remove_before:
|
|
n_backup -= 1
|
|
|
|
count = 0
|
|
|
|
if n_backup > self.keep:
|
|
logger.info('Removing old backups...')
|
|
dirs.sort()
|
|
|
|
n_remove = n_backup - self.keep
|
|
|
|
for i in range(n_remove):
|
|
logger.info('Removing backup %s (%d/%d)...', dirs[i], i + 1, n_remove)
|
|
|
|
if self._remove_backup(dirs[i]):
|
|
count += 1
|
|
|
|
self._removed_count = count
|
|
|
|
if count == 1:
|
|
logger.info('Removed %d backup', count)
|
|
elif count > 1:
|
|
logger.info('Removed %d backups', count)
|
|
|
|
def _list_backups(self) -> list[str] | None:
|
|
"""Names of the backup directories, or None if they can't be listed (then nothing is removed)"""
|
|
|
|
if self._remote:
|
|
assert self._ssh is not None
|
|
|
|
backup_dir = shlex.quote(self.output + '/simple_backup/')
|
|
|
|
# List only directories, and nothing (without errors) if no backup was made yet
|
|
_, stdout, stderr = self._ssh.exec_command(
|
|
f'if [ -d {backup_dir} ]; then find {backup_dir} -mindepth 1 -maxdepth 1 -type d; fi')
|
|
|
|
output = stdout.read().decode('utf-8')
|
|
err = stderr.read().decode('utf-8').strip()
|
|
|
|
if err != '':
|
|
# Don't risk removing the wrong directories if the list may be incomplete
|
|
logger.error('Cannot list old backups on the server. Old backups will not be removed')
|
|
logger.error(err)
|
|
|
|
return None
|
|
|
|
return [os.path.basename(d) for d in output.split('\n') if d != '']
|
|
|
|
try:
|
|
# Only count directories, so that stray files don't count as backups
|
|
return [f.name for f in os.scandir(f'{self.output}/simple_backup') if f.is_dir(follow_symlinks=False)]
|
|
except FileNotFoundError:
|
|
return None
|
|
except PermissionError:
|
|
logger.error('Cannot list old backups. Permission denied. Old backups will not be removed')
|
|
|
|
return None
|
|
|
|
def _remove_backup(self, name: str) -> bool:
|
|
"""Remove one backup directory; errors are logged"""
|
|
|
|
path = f'{self.output}/simple_backup/{name}'
|
|
|
|
if self._remote:
|
|
assert self._ssh is not None
|
|
|
|
sudo = 'sudo ' if self.remote_sudo else ''
|
|
_, _, stderr = self._ssh.exec_command(f'{sudo}rm -r {shlex.quote(path)}')
|
|
|
|
err = stderr.read().decode('utf-8').strip().split('\n')[0]
|
|
|
|
if err != '':
|
|
logger.error('Error while removing backup %s.', name)
|
|
logger.error(err)
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
try:
|
|
rmtree(path)
|
|
|
|
return True
|
|
except FileNotFoundError:
|
|
logger.error('Error while removing backup %s. Directory not found', name)
|
|
except PermissionError:
|
|
logger.error('Error while removing backup %s. Permission denied', name)
|
|
except OSError as e:
|
|
# E.g. read-only filesystem or device busy
|
|
logger.error('Error while removing backup %s. %s', name, e.strerror or e)
|
|
|
|
return False
|
|
|
|
def find_last_backup(self) -> None:
|
|
"""Get path of last backup (from last_backup symlink) for rsync --link-dest"""
|
|
|
|
if self._remote:
|
|
if self._ssh is None:
|
|
logger.critical('SSH connection to server failed')
|
|
sys.exit(5)
|
|
|
|
_, stdout, _ = self._ssh.exec_command(
|
|
f'find {shlex.quote(self.output + "/simple_backup/")} -mindepth 1 -maxdepth 1 -type d | sort')
|
|
output = stdout.read().decode('utf-8').strip().split('\n')
|
|
|
|
if output[-1] != '':
|
|
self._last_backup = output[-1]
|
|
else:
|
|
logger.info('No previous backups available')
|
|
else:
|
|
try:
|
|
dirs = sorted([f.path for f in os.scandir(
|
|
f'{self.output}/simple_backup') if f.is_dir(follow_symlinks=False)])
|
|
except FileNotFoundError:
|
|
logger.info('No previous backups available')
|
|
|
|
return
|
|
except PermissionError:
|
|
logger.critical(
|
|
'Cannot access the backup directory. Permission denied')
|
|
|
|
_notify('Backup failed (check log for details)')
|
|
|
|
sys.exit(3)
|
|
|
|
try:
|
|
self._last_backup = dirs[-1]
|
|
except IndexError:
|
|
logger.info('No previous backups available')
|
|
|
|
def _ssh_connect(self, homedir: str = '') -> paramiko.client.SSHClient | None:
|
|
try:
|
|
ssh = paramiko.SSHClient()
|
|
except NameError:
|
|
logger.error('Install paramiko for ssh support')
|
|
|
|
return None
|
|
|
|
connected = None
|
|
|
|
try:
|
|
connected = self._ssh_login(ssh, homedir)
|
|
|
|
return connected
|
|
finally:
|
|
# Don't leave a half-open connection (e.g. after failed authentication)
|
|
if connected is None:
|
|
ssh.close()
|
|
|
|
def _ssh_config(self, homedir: str) -> dict[str, Any]:
|
|
"""connect() arguments for ssh_host from ~/.ssh/config (HostName, Port, IdentityFile), as rsync's ssh uses it"""
|
|
|
|
assert self.ssh_host is not None
|
|
|
|
params: dict[str, Any] = {'hostname': self.ssh_host}
|
|
|
|
try:
|
|
host = paramiko.SSHConfig.from_path(f'{homedir}/.ssh/config').lookup(self.ssh_host)
|
|
|
|
params['hostname'] = host['hostname']
|
|
|
|
if 'port' in host:
|
|
params['port'] = int(host['port'])
|
|
except FileNotFoundError:
|
|
return params
|
|
except (OSError, paramiko.SSHException, ImportError, ValueError) as e:
|
|
# ImportError: 'Match exec' needs the invoke module
|
|
logger.warning('Cannot read %s/.ssh/config: %s', homedir, e)
|
|
|
|
return {'hostname': self.ssh_host}
|
|
|
|
# Like ssh, skip identity files that don't exist
|
|
keys = [k for k in host.get('identityfile', []) if os.path.isfile(k)]
|
|
|
|
if keys:
|
|
params['key_filename'] = keys
|
|
|
|
return params
|
|
|
|
def _ssh_login(self, ssh: paramiko.client.SSHClient, homedir: str) -> paramiko.client.SSHClient | None:
|
|
try:
|
|
ssh.load_host_keys(filename=f'{homedir}/.ssh/known_hosts')
|
|
except FileNotFoundError:
|
|
logger.warning('Cannot find file %s/.ssh/known_hosts', homedir)
|
|
|
|
ssh.set_missing_host_key_policy(paramiko.WarningPolicy())
|
|
|
|
host = self._ssh_config(homedir)
|
|
|
|
try:
|
|
# WarningPolicy emits a UserWarning for unknown host keys
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter('error', UserWarning)
|
|
ssh.connect(**host, username=self.ssh_user, timeout=SSH_TIMEOUT)
|
|
|
|
return ssh
|
|
except UserWarning:
|
|
try:
|
|
k = input(
|
|
f'Unknown key for host {self.ssh_host}. Continue anyway? (Y/N) ')
|
|
except EOFError:
|
|
# No terminal available (e.g. cron or systemd)
|
|
k = ''
|
|
|
|
if k.strip().upper().startswith('Y'):
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
else:
|
|
logger.critical('Unknown host key for %s. Connection aborted', self.ssh_host)
|
|
|
|
return None
|
|
except paramiko.BadHostKeyException as e:
|
|
logger.critical('Can\'t connect to the server.')
|
|
logger.critical(e)
|
|
|
|
return None
|
|
except paramiko.SSHException:
|
|
pass
|
|
|
|
try:
|
|
ssh.connect(**host, username=self.ssh_user, timeout=SSH_TIMEOUT)
|
|
|
|
return ssh
|
|
except paramiko.SSHException:
|
|
pass
|
|
|
|
if self.ssh_keyfile is None:
|
|
try:
|
|
# Fail instead of reading the password with echo if no terminal is available
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter('error', GetPassWarning)
|
|
password = getpass(
|
|
f'{self.ssh_user}@{self.ssh_host}\'s password: ')
|
|
|
|
ssh.connect(**host, username=self.ssh_user, timeout=SSH_TIMEOUT, password=password)
|
|
|
|
self._password = password
|
|
|
|
return ssh
|
|
except GetPassWarning as e:
|
|
logger.critical('Unable to get password')
|
|
logger.critical(e)
|
|
|
|
return None
|
|
except paramiko.SSHException as e:
|
|
logger.critical('Can\'t connect to the server.')
|
|
logger.critical(e)
|
|
|
|
return None
|
|
|
|
pkey = self._load_ssh_key()
|
|
|
|
if pkey is None:
|
|
return None
|
|
|
|
try:
|
|
ssh.connect(**host, username=self.ssh_user, timeout=SSH_TIMEOUT, pkey=pkey)
|
|
except paramiko.SSHException:
|
|
logger.critical('SSH connection to server failed')
|
|
|
|
return None
|
|
|
|
return ssh
|
|
|
|
def _load_ssh_key(self) -> paramiko.PKey | None:
|
|
try:
|
|
return paramiko.PKey.from_path(self.ssh_keyfile)
|
|
except TypeError:
|
|
# Raised when the key is encrypted and no passphrase was given
|
|
pass
|
|
except (OSError, ValueError, paramiko.UnknownKeyType) as e:
|
|
logger.critical('Unable to load SSH key %s', self.ssh_keyfile)
|
|
logger.critical(e)
|
|
|
|
return None
|
|
|
|
try:
|
|
# Fail instead of reading the passphrase with echo if no terminal is available
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter('error', GetPassWarning)
|
|
passphrase = getpass(
|
|
f'Enter passphrase for key \'{self.ssh_keyfile}\': ')
|
|
except GetPassWarning as e:
|
|
logger.critical('Unable to get passphrase')
|
|
logger.critical(e)
|
|
|
|
return None
|
|
|
|
try:
|
|
return paramiko.PKey.from_path(self.ssh_keyfile, passphrase.encode())
|
|
except (ValueError, paramiko.SSHException):
|
|
logger.critical('Unable to load SSH key %s, wrong passphrase?', self.ssh_keyfile)
|
|
|
|
return None
|
|
|
|
def _returncode_log(self, returncode: int) -> None:
|
|
match returncode:
|
|
case 2:
|
|
logger.error(
|
|
'Rsync error (return code 2) - Protocol incompatibility')
|
|
case 3:
|
|
logger.error(
|
|
'Rsync error (return code 3) - Errors selecting input/output files, dirs')
|
|
case 4:
|
|
logger.error(
|
|
'Rsync error (return code 4) - Requested action not supported')
|
|
case 5:
|
|
logger.error(
|
|
'Rsync error (return code 5) - Error starting client-server protocol')
|
|
case 10:
|
|
logger.error(
|
|
'Rsync error (return code 10) - Error in socket I/O')
|
|
case 11:
|
|
logger.error(
|
|
'Rsync error (return code 11) - Error in file I/O')
|
|
case 12:
|
|
logger.error(
|
|
'Rsync error (return code 12) - Error in rsync protocol data stream')
|
|
case 22:
|
|
logger.error(
|
|
'Rsync error (return code 22) - Error allocating core memory buffers')
|
|
case 23:
|
|
logger.warning(
|
|
'Rsync error (return code 23) - Partial transfer due to error')
|
|
case 24:
|
|
logger.warning(
|
|
'Rsync warning (return code 24) - Some source files vanished during the transfer')
|
|
case 30:
|
|
logger.error(
|
|
'Rsync error (return code 30) - Timeout in data send/receive')
|
|
case 35:
|
|
logger.error(
|
|
'Rsync error (return code 35) - Timeout waiting for daemon connection')
|
|
case _:
|
|
logger.error(
|
|
'Rsync error (return code %d) - Check rsync(1) for details', returncode)
|
|
|
|
# Function to read configuration file
|
|
@timing
|
|
def run(self) -> int:
|
|
"""Perform the backup"""
|
|
|
|
try:
|
|
return self._run()
|
|
finally:
|
|
self._remove_temp_files()
|
|
|
|
def close(self) -> None:
|
|
"""Close the SSH connection, if open"""
|
|
|
|
if self._ssh is not None:
|
|
self._ssh.close()
|
|
self._ssh = None
|
|
|
|
def _remove_temp_files(self) -> None:
|
|
for path in [self._inputs_path, self._exclude_path]:
|
|
if path != '':
|
|
try:
|
|
os.remove(path)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
def _run(self) -> int:
|
|
logger.info('Starting backup...')
|
|
|
|
_notify('Starting backup...')
|
|
|
|
# Check before anything is removed (--remove-before-backup)
|
|
rsync = which('rsync')
|
|
|
|
if rsync is None:
|
|
logger.critical('rsync not found. Install rsync to perform the backup')
|
|
|
|
_notify('Backup failed (check log for details)')
|
|
|
|
return 4
|
|
|
|
self.define_backup_dir()
|
|
|
|
fd, self._inputs_path = mkstemp(prefix='tmp_inputs', text=True)
|
|
count = 0
|
|
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as fp:
|
|
for i in self.inputs:
|
|
if not os.path.exists(i):
|
|
logger.warning('Input %s not found. Skipping', i)
|
|
else:
|
|
# rsync reads --files-from paths relative to the source ('/')
|
|
fp.write(os.path.abspath(i))
|
|
fp.write('\n')
|
|
count += 1
|
|
|
|
if count == 0:
|
|
logger.info(
|
|
'No existing files or directories specified for backup. Nothing to do')
|
|
|
|
_notify('Backup finished. No files copied')
|
|
|
|
return 1
|
|
|
|
fd, self._exclude_path = mkstemp(prefix='tmp_exclude', text=True)
|
|
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as fp:
|
|
for e in self.exclude:
|
|
fp.write(e)
|
|
fp.write('\n')
|
|
|
|
if self.keep != -1 and self._remove_before:
|
|
self.remove_old_backups()
|
|
|
|
# Only look for the last backup now, as it may have been removed above
|
|
self.find_last_backup()
|
|
|
|
logger.info('Copying files. This may take a long time...')
|
|
|
|
# Build the argument list directly, so paths with spaces or quotes are passed unchanged
|
|
args = [rsync, *self.options.split()]
|
|
|
|
if self._last_backup != '':
|
|
args.append(f'--link-dest={self._last_backup}')
|
|
|
|
args.extend([f'--exclude-from={self._exclude_path}', f'--files-from={self._inputs_path}',
|
|
'/', f'{self._server}{self._output_dir}'])
|
|
|
|
# None inherits our environment
|
|
env: dict[str, str] | None = None
|
|
|
|
# rsync splits the -e command itself, so quote the parts that may contain spaces
|
|
if self.ssh_keyfile is not None:
|
|
ssh = f'ssh -i {shlex.quote(self.ssh_keyfile)} -o StrictHostKeyChecking=accept-new'
|
|
elif self._password is not None and self.ssh_user is not None and which('sshpass'):
|
|
ssh = f'sshpass -e ssh -l {shlex.quote(self.ssh_user)} -o StrictHostKeyChecking=accept-new'
|
|
env = {**os.environ, 'SSHPASS': self._password}
|
|
else:
|
|
ssh = 'ssh -o StrictHostKeyChecking=accept-new'
|
|
|
|
args.extend(['-e', ssh])
|
|
|
|
if self._remote and self.remote_sudo:
|
|
args.append('--rsync-path=sudo rsync')
|
|
|
|
bar = None
|
|
|
|
if self._progress:
|
|
args.append('--info=progress2')
|
|
bar = _ProgressBar()
|
|
|
|
with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False, env=env) as p:
|
|
assert p.stdin is not None and isinstance(p.stdout, io.BufferedIOBase)
|
|
p.stdin.close()
|
|
|
|
# Without -v, only the summary (the last two lines) is logged
|
|
output: list[str] | deque[str] = [] if self._verbose else deque(maxlen=2)
|
|
|
|
try:
|
|
for r in _read_records(p.stdout):
|
|
if r != '' and (bar is None or not bar.update(r)):
|
|
output.append(r)
|
|
finally:
|
|
if bar is not None:
|
|
bar.clear()
|
|
|
|
returncode = p.wait()
|
|
|
|
if returncode == 0:
|
|
# Without -v, only the summary, which may be fewer than two lines (e.g. with --rsync-options q)
|
|
for r in output:
|
|
logger.info('rsync: %s', r)
|
|
else:
|
|
self._returncode_log(returncode)
|
|
|
|
if self._verbose:
|
|
level = logging.WARNING if returncode in [23, 24] else logging.ERROR
|
|
|
|
for r in output:
|
|
logger.log(level, 'rsync: %s', r)
|
|
|
|
if self._removed_count > 0 and returncode not in [0, 24]:
|
|
# With --remove-before-backup, old backups are already gone if the new one failed
|
|
if self.keep == 0:
|
|
message = 'All old backups were removed before the backup, and the new backup may be ' +\
|
|
'incomplete. No complete backup is available'
|
|
else:
|
|
message = 'Old backups were removed before the backup, and the new backup may be incomplete'
|
|
|
|
logger.error(message)
|
|
_notify(message)
|
|
|
|
if self.keep != -1 and not self._remove_before:
|
|
# Don't delete old backups if the new one may be incomplete
|
|
if returncode in [0, 24]:
|
|
self.remove_old_backups()
|
|
else:
|
|
logger.warning('Backup not completed successfully. Old backups will not be removed')
|
|
|
|
# Files vanishing during the transfer is normal for a live system (return code 24)
|
|
if returncode not in [0, 24]:
|
|
logger.error(
|
|
'Some errors occurred while performing the backup')
|
|
|
|
_notify(
|
|
'Some errors occurred while performing the backup. Check log for details')
|
|
|
|
return 4
|
|
|
|
logger.info('Backup completed')
|
|
|
|
_notify('Backup completed')
|
|
|
|
return 0
|
|
|
|
|
|
def _read_records(stream: io.BufferedIOBase) -> Iterator[str]:
|
|
"""Yield rsync output as it arrives, split on '\\n' and '\\r' (which ends progress updates)"""
|
|
|
|
buffer = b''
|
|
|
|
while chunk := stream.read1(65536):
|
|
*records, buffer = re.split(rb'[\r\n]', buffer + chunk)
|
|
|
|
for r in records:
|
|
yield r.decode('utf-8', errors='replace')
|
|
|
|
if buffer != b'':
|
|
yield buffer.decode('utf-8', errors='replace')
|
|
|
|
|
|
def _detect_user() -> str | None:
|
|
if os.geteuid() == 0:
|
|
return os.getenv('SUDO_USER') or os.getenv('DOAS_USER')
|
|
|
|
return os.getenv('USER')
|
|
|
|
|
|
def _parse_arguments() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(prog='simple_backup',
|
|
description='Simple backup script written in Python that uses rsync to copy files',
|
|
epilog='See simple_backup(1) manpage for full documentation',
|
|
formatter_class=MyFormatter)
|
|
|
|
parser.add_argument('-v', '--verbose', action='store_true',
|
|
help='More verbose output')
|
|
parser.add_argument('-c', '--config', default='~/.config/simple_backup/simple_backup.conf',
|
|
help='Specify location of configuration file')
|
|
parser.add_argument('-i', '--inputs', nargs='+',
|
|
help='Paths/files to backup')
|
|
parser.add_argument(
|
|
'-o', '--output', help='Output directory for the backup')
|
|
parser.add_argument('-e', '--exclude', nargs='+',
|
|
help='Files/directories/patterns to exclude from the backup')
|
|
parser.add_argument('-k', '--keep', type=int,
|
|
help='Number of old backups to keep')
|
|
parser.add_argument(
|
|
'-u', '--user', help='Explicitly specify the user running the backup')
|
|
parser.add_argument('-s', '--checksum', action='store_true',
|
|
help='Use checksum rsync option to compare files')
|
|
parser.add_argument(
|
|
'--ssh-host', help='Server hostname (for remote backup)')
|
|
parser.add_argument(
|
|
'--ssh-user', help='Username to connect to server (for remote backup)')
|
|
parser.add_argument('--keyfile', help='SSH key location')
|
|
parser.add_argument('-z', '--compress', action='store_true',
|
|
help='Compress data during the transfer')
|
|
parser.add_argument('--remove-before-backup', action='store_true',
|
|
help='Remove old backups before executing the backup, instead of after')
|
|
parser.add_argument('--no-syslog', action='store_true',
|
|
help='Disable systemd journal logging')
|
|
parser.add_argument('--no-progress', action='store_true',
|
|
help='Don\'t show a progress bar (it is only shown on a terminal)')
|
|
parser.add_argument('--rsync-options', nargs='+',
|
|
choices=['a', 'l', 'p', 't', 'g', 'o',
|
|
'c', 'h', 's', 'D', 'H', 'X'],
|
|
help='Specify options for rsync')
|
|
parser.add_argument('--remote-sudo', action='store_true',
|
|
help='Run rsync on remote server with sudo if allowed')
|
|
parser.add_argument('--numeric-ids', action='store_true',
|
|
help='Use rsync \'--numeric-ids\' option (don\'t map uid/gid values by name)')
|
|
parser.add_argument('--last-backup', action='store_true',
|
|
help='Show when the last successful backup was made and exit')
|
|
|
|
args = parser.parse_args()
|
|
|
|
return args
|
|
|
|
|
|
def _expand_home(path: str, user: str) -> str:
|
|
"""Expand a leading '~' to the home directory of user (other '~' are left as they are)"""
|
|
|
|
if path == '~' or path.startswith('~/'):
|
|
path = f'~{user}{path[1:]}'
|
|
|
|
return os.path.expanduser(path)
|
|
|
|
|
|
def _expand_inputs(inputs: list[str], user: str | None = None) -> list[str]:
|
|
expanded_inputs: list[str] = []
|
|
|
|
for i in inputs:
|
|
if i == '':
|
|
continue
|
|
|
|
if user is not None:
|
|
path = _expand_home(i, user)
|
|
else:
|
|
path = i
|
|
|
|
if i.startswith('~'):
|
|
logger.warning('Cannot expand \'~\'. No user specified')
|
|
|
|
# An existing path is used as is, so names with glob characters (e.g. 'Photos [2020]') still work
|
|
i_ex = [path] if os.path.lexists(path) else glob(path)
|
|
|
|
if len(i_ex) == 0:
|
|
logger.warning(
|
|
'No file or directory matching input %s. Skipping...', i)
|
|
else:
|
|
expanded_inputs.extend(i_ex)
|
|
|
|
return expanded_inputs
|
|
|
|
|
|
class ConfigArgs(TypedDict):
|
|
"""Values read from the configuration file"""
|
|
|
|
inputs: list[str] | None
|
|
output: str | None
|
|
exclude: list[str] | None
|
|
keep: int
|
|
ssh_host: str | None
|
|
ssh_user: str | None
|
|
ssh_keyfile: str | None
|
|
remote_sudo: bool
|
|
numeric_ids: bool
|
|
|
|
|
|
def _get_bool(config: configparser.ConfigParser, key: str) -> bool:
|
|
# Empty values (as in the example config) count as false
|
|
if config.get('server', key, fallback='').strip() == '':
|
|
return False
|
|
|
|
return config.getboolean('server', key)
|
|
|
|
|
|
def _read_config(config_file: str, user: str | None = None) -> ConfigArgs:
|
|
config_args: ConfigArgs = {'inputs': None,
|
|
'output': None,
|
|
'exclude': None,
|
|
'keep': -1,
|
|
'ssh_host': None,
|
|
'ssh_user': None,
|
|
'ssh_keyfile': None,
|
|
'remote_sudo': False,
|
|
'numeric_ids': False}
|
|
|
|
if not os.path.isfile(config_file):
|
|
if user is not None:
|
|
logger.warning('Config file %s does not exist', config_file)
|
|
else:
|
|
logger.warning(
|
|
'User not specified. Can\'t read configuration file')
|
|
|
|
return config_args
|
|
|
|
# Disable interpolation, so that '%' can be used in paths and patterns
|
|
config = configparser.ConfigParser(interpolation=None)
|
|
config.read(config_file)
|
|
|
|
# Allow compatibility with previous version of config file
|
|
if config.has_section('backup'):
|
|
section = 'backup'
|
|
elif config.has_section('default'):
|
|
section = 'default'
|
|
else:
|
|
raise configparser.NoSectionError('backup')
|
|
|
|
# inputs and backup_dir are optional, since -i and -o can give them. Missing ones are reported by check_params()
|
|
inputs = config.get(section, 'inputs', fallback=None)
|
|
|
|
if inputs is not None:
|
|
config_args['inputs'] = list(set(_expand_inputs([i.strip() for i in inputs.split(',')], user)))
|
|
|
|
# '~' is expanded later, since for remote backups it refers to the home on the server
|
|
config_args['output'] = config.get(section, 'backup_dir', fallback=None)
|
|
|
|
exclude = config.get(section, 'exclude', fallback='')
|
|
config_args['exclude'] = [e.strip() for e in exclude.split(',') if e.strip() != '']
|
|
|
|
config_args['keep'] = config.getint(section, 'keep', fallback=-1)
|
|
|
|
# Read host and user separately, so that a missing one is reported instead of silently
|
|
# disabling remote backup. Empty values count as not set
|
|
config_args['ssh_host'] = config.get('server', 'ssh_host', fallback='').strip() or None
|
|
config_args['ssh_user'] = config.get('server', 'ssh_user', fallback='').strip() or None
|
|
|
|
config_args['ssh_keyfile'] = config.get('server', 'ssh_keyfile', fallback='').strip() or None
|
|
|
|
config_args['remote_sudo'] = _get_bool(config, 'remote_sudo')
|
|
config_args['numeric_ids'] = _get_bool(config, 'numeric_ids')
|
|
|
|
return config_args
|
|
|
|
|
|
def _target_uid() -> int | None:
|
|
euid = os.geteuid()
|
|
|
|
if euid != 0:
|
|
return euid
|
|
|
|
if (val := os.getenv('SUDO_UID')) is not None:
|
|
return int(val)
|
|
|
|
if (name := os.getenv('DOAS_USER')) is not None:
|
|
try:
|
|
return pwd.getpwnam(name).pw_uid
|
|
except KeyError:
|
|
return None
|
|
|
|
return None
|
|
|
|
|
|
_notify_failed = False
|
|
|
|
|
|
def _notify(text: str) -> None:
|
|
global _notify_failed
|
|
|
|
if dbus is None or _notify_failed:
|
|
return
|
|
|
|
euid = os.geteuid()
|
|
uid = _target_uid()
|
|
|
|
if uid is None:
|
|
logger.debug('Cannot send desktop notification: user not found')
|
|
|
|
return
|
|
|
|
os.seteuid(uid)
|
|
|
|
try:
|
|
os.environ['DBUS_SESSION_BUS_ADDRESS'] = f'unix:path=/run/user/{uid}/bus'
|
|
|
|
obj = dbus.SessionBus().get_object('org.freedesktop.Notifications',
|
|
'/org/freedesktop/Notifications')
|
|
obj = dbus.Interface(obj, 'org.freedesktop.Notifications')
|
|
obj.Notify('simple_backup', 0, '', 'simple_backup', text, [], {'urgency': dbus.Byte(1)}, 10000)
|
|
except Exception as e:
|
|
# Report the failure only once (e.g. no session bus available), then stop trying
|
|
_notify_failed = True
|
|
logger.warning('Cannot send desktop notification: %s', e)
|
|
finally:
|
|
os.seteuid(euid)
|
|
|
|
|
|
def _state_owner(user: str | None) -> pwd.struct_passwd | None:
|
|
"""The user whose state file is used when running as root on their behalf (sudo, doas, -u)"""
|
|
|
|
if os.geteuid() != 0 or user is None:
|
|
return None
|
|
|
|
try:
|
|
owner = pwd.getpwnam(user)
|
|
except KeyError:
|
|
return None
|
|
|
|
return owner if owner.pw_uid != 0 else None
|
|
|
|
|
|
def _state_file(owner: pwd.struct_passwd | None) -> str:
|
|
if owner is not None:
|
|
# XDG_STATE_HOME belongs to root here
|
|
base = os.path.join(owner.pw_dir, '.local', 'state')
|
|
else:
|
|
base = os.getenv('XDG_STATE_HOME', '')
|
|
|
|
# The XDG spec says to ignore relative paths
|
|
if not os.path.isabs(base):
|
|
base = os.path.expanduser('~/.local/state')
|
|
|
|
return os.path.join(base, 'simple_backup', 'last_backup')
|
|
|
|
|
|
def _write_state(user: str | None, destination: str) -> None:
|
|
"""Record the time and directory of the last successful backup (two lines)"""
|
|
|
|
owner = _state_owner(user)
|
|
path = _state_file(owner)
|
|
tmp = ''
|
|
euid, egid = os.geteuid(), os.getegid()
|
|
|
|
# Create the file (and any missing directories) as the user, not root
|
|
if owner is not None:
|
|
os.setegid(owner.pw_gid)
|
|
os.seteuid(owner.pw_uid)
|
|
|
|
try:
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
|
|
# Replace the file in one step, so readers (e.g. status bar scripts) never see it half written
|
|
fd, tmp = mkstemp(prefix='.last_backup', dir=os.path.dirname(path), text=True)
|
|
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as fp:
|
|
fp.write(f'{datetime.now().astimezone().isoformat(timespec="seconds")}\n{destination}\n')
|
|
|
|
os.replace(tmp, path)
|
|
except OSError as e:
|
|
logger.warning('Cannot save the time of the last backup to %s: %s', path, e)
|
|
|
|
if tmp != '':
|
|
try:
|
|
os.remove(tmp)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
if owner is not None:
|
|
os.seteuid(euid)
|
|
os.setegid(egid)
|
|
|
|
|
|
def _show_last_backup(user: str | None) -> int:
|
|
path = _state_file(_state_owner(user))
|
|
|
|
try:
|
|
with open(path, encoding='utf-8') as fp:
|
|
time, destination = fp.read().splitlines()[:2]
|
|
|
|
completed = datetime.fromisoformat(time).astimezone()
|
|
except FileNotFoundError:
|
|
print('No backup recorded yet')
|
|
|
|
return 1
|
|
except (OSError, ValueError) as e:
|
|
print(f'Cannot read {path}: {e}', file=sys.stderr)
|
|
|
|
return 1
|
|
|
|
days = (datetime.now().astimezone().date() - completed.date()).days
|
|
|
|
if days == 0:
|
|
age = 'today'
|
|
elif days == 1:
|
|
age = 'yesterday'
|
|
else:
|
|
age = f'{days} days ago'
|
|
|
|
print(f'Last backup: {completed:%Y-%m-%d %H:%M} ({age})')
|
|
print(f'Directory: {destination}')
|
|
|
|
return 0
|
|
|
|
|
|
def simple_backup() -> int:
|
|
"""Main"""
|
|
|
|
args = _parse_arguments()
|
|
|
|
user: str | None = args.user or _detect_user()
|
|
|
|
if args.last_backup:
|
|
return _show_last_backup(user)
|
|
|
|
if not args.user and os.geteuid() != 0:
|
|
homedir = os.getenv('HOME', '')
|
|
elif user is not None:
|
|
homedir = _expand_home('~', user)
|
|
else:
|
|
logger.warning('Failed to detect user. You can use -u/--user parameter to manually specify it')
|
|
homedir = ''
|
|
|
|
if args.no_syslog:
|
|
try:
|
|
logger.removeHandler(j_handler)
|
|
except NameError:
|
|
pass
|
|
|
|
# Expand '~' to the home of the user running the backup, not root's
|
|
config_file = _expand_home(args.config, user) if user is not None else os.path.expanduser(args.config)
|
|
|
|
try:
|
|
config_args = _read_config(config_file, user)
|
|
except (configparser.Error, ValueError) as e:
|
|
# configparser.Error: missing sections/options or bad syntax, ValueError: invalid values (e.g. keep=abc)
|
|
logger.critical('Bad configuration file: %s', e)
|
|
|
|
return 6
|
|
|
|
inputs = args.inputs if args.inputs is not None else config_args['inputs']
|
|
exclude = args.exclude if args.exclude is not None else config_args['exclude']
|
|
keep = args.keep if args.keep is not None else config_args['keep']
|
|
ssh_host = args.ssh_host if args.ssh_host is not None else config_args['ssh_host']
|
|
ssh_user = args.ssh_user if args.ssh_user is not None else config_args['ssh_user']
|
|
|
|
if keep < -1:
|
|
logger.critical('Invalid number of backups to keep: %d (use -1 to keep all)', keep)
|
|
|
|
return 6
|
|
|
|
if (ssh_host is None) != (ssh_user is None):
|
|
logger.critical('Both SSH host and SSH user are needed for remote backup')
|
|
|
|
return 6
|
|
|
|
if args.output is not None:
|
|
output = args.output
|
|
else:
|
|
output = config_args['output']
|
|
|
|
# For remote backups, '~' is expanded on the server
|
|
remote = ssh_host is not None and ssh_user is not None
|
|
|
|
if output is not None and not remote:
|
|
if user is not None:
|
|
output = _expand_home(output, user)
|
|
elif output.startswith('~'):
|
|
logger.warning('Cannot expand \'~\', no user specified')
|
|
|
|
ssh_keyfile = args.keyfile if args.keyfile is not None else config_args['ssh_keyfile']
|
|
remote_sudo = args.remote_sudo or config_args['remote_sudo']
|
|
|
|
# Options needed in any case (--mkpath creates the backup directory on the first run)
|
|
rsync_options = ['-r', '-v', '--ignore-missing-args', '--mkpath']
|
|
|
|
if args.rsync_options is None:
|
|
rsync_options.extend(['-a', '-h', '-H', '-X', '-s'])
|
|
else:
|
|
for ro in args.rsync_options:
|
|
rsync_options.append(f'-{ro}')
|
|
|
|
if args.checksum:
|
|
rsync_options.append('-c')
|
|
|
|
if args.compress:
|
|
rsync_options.append('-z')
|
|
|
|
if args.numeric_ids or config_args['numeric_ids']:
|
|
rsync_options.append('--numeric-ids')
|
|
|
|
backup = Backup(inputs, output, exclude, keep, ' '.join(rsync_options), ssh_host, ssh_user, ssh_keyfile,
|
|
remote_sudo, remove_before=args.remove_before_backup, verbose=args.verbose,
|
|
progress=not args.no_progress and sys.stderr.isatty())
|
|
|
|
try:
|
|
return_code = backup.check_params(homedir)
|
|
|
|
if return_code != 0:
|
|
return return_code
|
|
|
|
return_code = backup.run()
|
|
|
|
if return_code == 0:
|
|
_write_state(user, backup.destination)
|
|
|
|
return return_code
|
|
finally:
|
|
backup.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(simple_backup())
|