69 Commits
3.1.1 ... 4.0.0

Author SHA1 Message Date
a26182cedd Update version number 2023-06-16 19:16:47 +02:00
5b3558c282 Merge branch 'development' 2023-06-16 19:15:44 +02:00
4e6adf3c56 Use dict for config options 2023-06-16 17:42:13 +02:00
ffed2dec90 Add numeric-ids option for rsync 2023-06-16 16:18:12 +02:00
169f824d83 Improve documentation 2023-06-15 23:15:00 +02:00
f77ff2d24f Allow running remote rsync as sudo 2023-06-15 23:12:19 +02:00
ffe2b69c99 Fix error in man page 2023-06-15 22:12:38 +02:00
82b0ea88fa Merge branch 'master' into development 2023-06-15 22:10:19 +02:00
970d3dde1e Add manual selection of rsync options 2023-06-15 21:30:43 +02:00
4d23bde906 Check that inputs from command line exist
Check that the inputs specified on the command line (i.e. with the
option '-i' or '--input') exist and print a warning when they don't.
If no valid inputs are found, exit.
2023-06-15 18:44:22 +02:00
ff70358496 Version bump 2023-06-15 17:38:00 +02:00
6f1e91e2cd Add more return codes 2023-06-15 17:14:17 +02:00
b3fee0d022 Add some meaningful return codes 2023-06-15 16:58:56 +02:00
d63eb8f771 Improve handling of missing inputs 2023-06-15 15:34:00 +02:00
f95cd86cdc Check backup folder for remote backup
Check that the backup folder exists at the end of the backup when
performing backup over ssh
2023-06-15 11:48:23 +02:00
f37cd89b4e Fix parsing of old config file 2023-06-15 09:30:59 +02:00
d1b429d37a Merge branch 'master' into development 2023-06-04 16:53:22 +02:00
56df958c5b Add expansion of params in config file
Allow using wildcards (i.e. * to match any character and ~ to match the
user's home directory) in inputs and ouput variables in config file
2023-06-04 12:09:30 +02:00
d6d9fbf6e4 Add no-syslog option 2023-06-04 10:16:50 +02:00
cee4d13837 Fix bug
Fix crash when attempting to close a non-existent paramiko
SSH connection
2023-06-03 16:14:36 +02:00
252617e4f2 Fix old backups counting 2023-06-03 16:09:34 +02:00
809545b172 Explicitly close paramiko connection 2023-06-03 15:56:24 +02:00
b34627fe58 Improve readability 2023-06-03 15:44:31 +02:00
cd558e8608 Merge branch 'nolink' into development 2023-06-02 20:12:53 +02:00
ffbf8ece91 Remove last_backup link 2023-06-02 19:38:28 +02:00
78760bdda3 Merge branch 'master' into development 2023-06-02 18:38:54 +02:00
6c07c147ae Version bump 2023-06-02 17:04:53 +02:00
5ddd374350 Fix notification bug
Correctly handle exception when trying to display notification if
dbus-python is not available
2023-06-02 17:01:37 +02:00
3d3fbbcbd9 Add password authentication for SSH 2023-06-02 00:09:14 +02:00
4f3d83f458 Update manpage 2023-06-01 22:18:32 +02:00
d48eaabbd3 Merge branch 'master' into development 2023-06-01 17:47:44 +02:00
525f381094 Remove PKGBUILD
Removed PKGBUILD since the packages is available in the AUR. Added
a note about AUR in README.md
2023-06-01 17:23:58 +02:00
e1ba388296 Update version 2023-06-01 17:18:42 +02:00
e84cf81b13 Add man page 2023-06-01 17:08:31 +02:00
7664fa1b33 Update README.md 2023-05-31 22:12:59 +02:00
98cb7f5822 Fix ssh authentication when running with sudo 2023-05-31 20:39:03 +02:00
b957200deb Change missing hostkey policy 2023-05-31 19:30:31 +02:00
9e90d620e6 Merge branch 'master' into development 2023-05-31 19:21:11 +02:00
06620a4dba Fix bug when exclude pattern is None 2023-05-31 19:07:50 +02:00
ba97b25e25 Add docstrings 2023-05-29 23:10:29 +02:00
335ad348e5 Update README.md 2023-05-29 21:56:58 +02:00
df18e383ed Update dependencies 2023-05-29 19:02:05 +02:00
7eb71bc924 Add rsync compress option 2023-05-29 18:33:02 +02:00
c25ef52393 Implement remove old backups from server 2023-05-29 17:57:12 +02:00
24a59bde2d Use ssh agent if available 2023-05-29 00:09:54 +02:00
88e6a9a141 Add incremental backups on server 2023-05-28 23:19:08 +02:00
38c090e257 Add basic remote backup functionality 2023-05-28 21:30:40 +02:00
0dd7b887f7 Update PKGBUILD to use release version 2023-05-28 11:07:03 +02:00
7684bc43b0 Add desktop notifications 2023-05-25 23:44:59 +02:00
a07fce1b6c Version bump 2023-05-25 19:01:50 +02:00
11c1ab0952 Add options to remove old backups before copying files 2023-05-25 19:01:19 +02:00
a42ade4f2b Version bump 2023-05-25 17:27:04 +02:00
c6432c1350 Check rsync return code 2023-05-25 17:20:32 +02:00
fa1d8f410e Remove old last_backup link after copying files 2023-05-25 17:10:38 +02:00
8b9087cdf6 Version bump 2023-05-11 20:21:00 +02:00
3c155e62a2 Version bump 2023-05-11 20:19:56 +02:00
5a4a26f9a7 Fix minor bugs and improve exception handling 2023-05-11 20:18:42 +02:00
ca0e3d133f Fix entry_point in setup.cfg 2023-05-05 19:41:32 +02:00
cc788735dd Fix README.md 2023-05-05 19:30:53 +02:00
631ffa85d3 Remove broken desktop notifications 2023-05-05 19:23:21 +02:00
fe2d66c24c Remove data_files from setup.cfg 2023-05-05 19:12:41 +02:00
97419c30f9 Use setuptools to build the project 2023-05-05 19:03:23 +02:00
c6871ac81a Bump PKGBUILD version 2023-05-04 23:42:55 +02:00
90120031e2 Update PKGBUILD 2023-05-04 23:21:12 +02:00
d61ffa199a Use setuptools 2023-05-04 23:16:15 +02:00
a4c4b88193 Fix PKGBUILD 2023-05-02 18:10:43 +02:00
eb8bdde1fc Rename configuration file 2023-05-02 18:07:41 +02:00
5d17aaf03a Update dependencies in PKGBUILD 2023-04-30 22:06:30 +02:00
d181d2970f Better handle missing parameters 2023-04-30 14:38:26 +02:00
13 changed files with 1068 additions and 380 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
.idea/ .idea/
__pycache__/
test/ test/

View File

@ -1,32 +0,0 @@
#Arch Linux PKGBUILD
#
#Maintainer: Daniele Fucini <dfucini@gmail.com>
#
pkgname=simple_backup
pkgver=2.0.0.r3.gc61c704
pkgrel=1
pkgdesc='Simple backup script that uses rsync to copy files'
arch=('any')
url="https://github.com/Fuxino/simple_backup.git"
license=('GPL3')
makedepends=('git')
depends=('python3'
'rsync'
'python-dotenv'
'python-dbus')
install=${pkgname}.install
source=(git+https://github.com/Fuxino/${pkgname}.git)
sha256sums=('SKIP')
pkgver()
{
cd "$pkgname"
git describe --long --tags | sed 's/\([^-]*-g\)/r\1/;s/-/./g'
}
package()
{
install -Dm755 "${srcdir}/${pkgname}/${pkgname}.py" "${pkgdir}/usr/bin/${pkgname}"
install -Dm644 "${srcdir}/${pkgname}/${pkgname}.config" "${pkgdir}/etc/${pkgname}/${pkgname}.config"
}

View File

@ -1,10 +1,71 @@
# simple_backup simple_backup
============
A simple backup script A simple backup script
## Description ## Description
simple_backup is a Python script that allows you to backup your files. simple_backup is a Python script that allows you to backup your files.
It reads from a configuration file the files/directories that must be copied, Parameters like input files/directories, output directory etc. can be specified in a configuration file, or on the command line.
the destination directory for the backup and a few other options. Run:
```bash
simple_backup -h
```
to print all possible command line options.
## Dependencies ## Dependencies
rsync is used to perform the backup. The script uses rsync to actually run the backup, so you will have to install it on your system. For example on Arch Linux:
```bash
sudo pacman -Syu rsync
```
It's also required to have python-dotenv
Optional dependencies are systemd-python to enable using systemd journal for logging, and dbus-python for desktop notifications.
## Install
To install the program, first clone the repository:
```bash
git clone https://github.com/Fuxino/simple_backup.git
```
Then install the tools required to build the package:
```bash
pip install --upgrade build wheel
```
Finally, run:
```bash
cd simple_backup
python -m build --wheel
python -m pip install dist/*.whl
```
For Arch Linux and Arch-based distros, two packages are available in the AUR (aur.archlinux.org):
- **simple_backup** for the release version
- **simple_backup-git** for the git version
## Remote backup
> **Warning**
> This feature is experimental
It's possible to use a remote server as destination for the backup. Just use the --username (or -u) and --host arguments (or set them in the configuration file).
For this to work, rsync must be installed on the server too.
### Server authentication
The best way to handle the authentication is to have an ssh agent running on your system, otherwise if a passphrase is necessary to unlock the ssh key, it will be necessary to enter it more than once.
If needed, it's possible to specify the ssh key location with the --keyfile argument or in the configuration file.
To be able to connect to the user's ssh agent when running simple_backup with sudo, make sure to preserve the SSH_AUTH_SOCK environment variable. For example:
```bash
sudo --preserve-env=SSH_AUTH_SOCK -s simple_backup [options]
```
or by editing the sudoers file.
If SSH key authentication is not available, password authentication will be used instead.

180
man/simple_backup.1 Normal file
View File

@ -0,0 +1,180 @@
.TH SIMPLE_BACKUP 1 2023-06-15 SIMPLE_BACKUP 3.2.6
.SH NAME
simple_backup \- Backup files and folders using rsync
.SH SYNOPSIS
.BR simple_backup
\-h, \-\-help
.PD 0
.P
.PD
.BR simple_backup
[\-c, \-\-config FILE]
[\-i, \-\-input INPUT [INPUT...]]
[\-o, \-\-output DIR]
.PD 0
.P
.PD
.RS 14 [\-e, \-\-exclude FILE|DIR|PATTERN [FILE|...]]
[\-k, \-\-keep N]
[\-\-host HOSTNAME]
[\-u, \-\-username USERNAME]
[\-\-keyfile FILE]
.PD 0
.P
.PD
[\-s, \-\-checksum]
[\-z, \-\-compress]
[\-\-remove\-before\-backup]
.RE
.SH DESCRIPTION
.BR simple_backup
is a python script for performing backup of files and folders.
.P
It uses rsync to copy the files to the specified location. Parameters for the backup such as
input files/directories, output location and files or folders to exclude can be specified
in a configuration file (default location $HOME/.config/simple_backup/simple_backup.conf)
or directly on the command line.
.P
Parameters specified on the command line will override those in the configuration file.
.SH OPTIONS
.TP
.B \-h, \-\-help
Print a short help message and exit
.TP
.B \-c, \-\-config FILE
Specify the configuration file, useful to specify a different one from the default.
.TP
.B \-i, \-\-input INPUT [INPUT...]
Specify the files and directories to backup. Multiple inputs can be specified, just separate
them with a space. If filenames or paths contain spaces, don't forget to escape them,
or to use single or double quotes around them.
.TP
.B \-o, \-\-output DIR
Specify the directory where the files will be copied. The program will automatically
create a subdirectory called \(aqsimple_backup\(aq (if it does not already exist) and
inside this directory the actual backup directory (using the current date and time)
.TP
.B \-e, \-\-exclude FILE|DIR|PATTERN [FILE|...]]
Specify files, directories or patterns to exclude from the backup. Matching files and directories
will not be copied. Multiple elements can be specified, in the same way as for the \-\-input option
.TP
.B \-k, \-\-keep N
Specify how many old backups (so excluding the current one) will be kept. The default behavior
is to keep them all (same as N=\-1)
.TP
.B \-\-host HOSTNAME
Hostname of the server where to copy the files in case of remote backup through SSH
.TP
.B \-u, \-\-username USERNAME
Username for connecting to the server in case of remote backup
.TP
.B \-\-keyfile FILE
Location of the SSH key for server authentication.
.TP
.B \-s, \-\-checksums
Same as rsync option \(aq\-\-checksum\(aq, use checksums instead of mod\-time and size
to skip files.
.TP
.B \-z, \-\-compress
Compress data during transfer (rsync \(aq\-\-compress\(aq option). Useful for remote backup
if saving bandwith is needed.
.TP
.B \-\-remove\-before\-backup
Remove old backups (if necessary) before creating the new backup. Useful to free some space
before performing the backup.
Default behavior is to remove old backups after successfully completing the backup.
.TP
.B \-\-no\-syslog
Don't use systemd journal for logging
.TP
.B \-\-rsync\-options OPTIONS [OPTION...]
By default, the following rsync options are used:
.RS
.PP
\-a \-r \-v \-h \-s \-H \-X
.PP
Using \-\-rsync\-options it is possible to manually select which options to use. Supported values are the following:
.PP
\-a, \-l, \-p, \-t, \-g, \-o, \-c, \-h, \-D, \-H, \-X, \-s
.PP
Options \-r and \-v are used in any case. Not that options must be specified without dash (\-), for example:
.PP
.EX
simple_backup \-\-rsync\-options a l p
.EE
.TP
Check
.B rsync (1)
for details about the options.
.RE
.TP
.B \-\-remote\-sudo
Run rsync on the remote server with sudo. This is needed if you want to preserve the owner of the files/folders to be copied (rsync \-\-owner option). For this to work the user used to login to the server obviously need to be allowed to use sudo. In addition, the user need to be able to run rsync with sudo without a password. To do this, /etc/sudoers on the server need to be edited adding a line like this one:
.RS
.PP
<username> ALL=NOPASSWD:<path/to/rsync>
.RE
.TP
.B \-\-numeric\-ids
Use rsync \-\-numeric\-ids option. This causes rsync to use numeric uid/gid instead of trying to map uid/gid names from the local machine to the server
.SH CONFIGURATION
An example configuration file is provided at \(aq/usr/share/doc/simple_backup/simple_backup.conf\(aq.
Copy it to the default location ($HOME/.config/simple_backup) and edit it as needed.
.SH REMOTE BACKUP
It is possible to choose a directory on a remote server as destination for the backup. The files
are copied by rsync through SSH. Server hostname and username must be specified, either in the
configuration file, or on the command line (\(aq\-\-host\(aq and \(aq\-\-username\(aq options).
.SS AUTHENTICATION
For authentication, it is possible to use SSH key or password.
.P
When using SSH key, the best way to connect to the server is to have an SSH agent running.
Otherwise, if the SSH key is encrypted, it will be necessary to enter the passphrase more
than once. It is possible to specify the SSH key to use with the option \(aq\-\-keyfile\(aq,
if necessary.
.P
When running
.B simple_backup
with
.B sudo,
in order to connect to the user\(aq s SSH agent it is necessary to preserve the \(aq SSH_AUTH_SOCK\(aq environment variable, for example:
.P
.EX
sudo --preserve-env=SSH_AUTH_SOCK -s simple_backup [options]
.EE
.P
It is also possible to make this permanent by editing the
.B sudoers
file (see
.B sudoers (5)
)
.P
If SSH key authentication is not available, password authentication will be used instead.
Note that in this case
.B sshpass
(if available) will be used to send the password to rsync, to avoid prompting the user for
the password multiple
times. This can pose some security risks, see
.B sshpass (1)
for details. For this reason, use SSH key authentication if possible.
.SH EXIT STATUS
.TP
.B 0
The backup was completed without errors
.TP
.B 1
No valid inputs selected for backup
.TP
.B 2
Backup failed because output directory for storing the backup does not exist
.TP
.B 3
Permission denied to access the output directory
.TP
.B 4
rsync error (rsync returned a non-zero value)
.SH SEE ALSO
.BR rsync (1)
.SH AUTHORS
.MT https://github.com/Fuxino
Daniele Fucini
.ME

3
pyproject.toml Normal file
View File

@ -0,0 +1,3 @@
[build-system]
requires = ['setuptools']
build-backend = 'setuptools.build_meta'

View File

@ -1 +0,0 @@
python-dotenv>=1.0.0

40
setup.cfg Normal file
View File

@ -0,0 +1,40 @@
[metadata]
name = simple_backup
version = attr: simple_backup.__version__
description = Simple backup script using rsync
long_description = file: README.md
author = Daniele Fucini
author_email = dfucini@gmail.com
license = GPL3
url = https://github.com/Fuxino/simple_backup
classifiers =
Development Status :: 4 - Beta
Environment :: Console
License :: OSI Approved :: GNU General Public License v3 (GPLv3)
Natural Language :: English
Operating System :: POSIX :: Linux
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
Topic :: System :: Archiving :: Backup
[options]
packages = simple_backup
python_requires = >=3.7
install_requires =
python-dotenv
[options.extras_require]
JOURNAL =
systemd-python
NOTIFICATIONS =
dbus-python
REMOTE =
paramiko
[options.entry_points]
console_scripts =
simple_backup = simple_backup.simple_backup:simple_backup

View File

@ -1,14 +0,0 @@
#Example config file for simple_backup
[default]
#Input directories. Use a comma to separate items
inputs=/home/my_home,/etc
#Output directory
backup_dir=/media/Backup
#Exclude patterns. Use a comma to separate items
exclude=.gvfs,.local/share/gvfs-metadata,.cache,.dbus,.Trash,.local/share/Trash,.macromedia,.adobe,.recently-used,.recently-used.xbel,.thumbnails
#Number of snapshots to keep (use -1 to keep all)
keep=-1

View File

@ -1,8 +0,0 @@
post_install() {
echo "An example configuration file is found in /etc/simple_backup/simple_backup.config."
echo "Copy it to ~/config/simple_backup/simple_backup.config and modify it as needed"
}
post_upgrade() {
post_install
}

View File

@ -1,321 +0,0 @@
#!/usr/bin/python3
# Import libraries
import os
from functools import wraps
from shutil import rmtree
import argparse
import configparser
import logging
from logging import StreamHandler
from timeit import default_timer
from subprocess import Popen, PIPE, STDOUT
from datetime import datetime
from tempfile import mkstemp
import dbus
from dotenv import load_dotenv
try:
from systemd import journal
except ImportError:
pass
load_dotenv()
euid = os.geteuid()
if euid == 0:
user = os.getenv("SUDO_USER")
homedir = os.path.expanduser(f'~{user}')
else:
homedir = os.getenv('HOME')
logging.getLogger().setLevel(logging.DEBUG)
logger = logging.getLogger(os.path.basename(__file__))
c_handler = StreamHandler()
c_handler.setLevel(logging.INFO)
c_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
c_handler.setFormatter(c_format)
logger.addHandler(c_handler)
try:
j_handler = journal.JournalHandler()
j_handler.setLevel(logging.INFO)
j_format = logging.Formatter('%(levelname)s - %(message)s')
j_handler.setFormatter(j_format)
logger.addHandler(j_handler)
except NameError:
pass
def timing(_logger):
def decorator_timing(func):
@wraps(func)
def wrapper_timing(*args, **kwargs):
start = default_timer()
value = func(*args, **kwargs)
end = default_timer()
_logger.info(f'Elapsed time: {end - start:.3f} seconds')
return value
return wrapper_timing
return decorator_timing
class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
pass
class Backup:
def __init__(self, inputs, output, exclude, keep, options):
self.inputs = inputs
self.output = output
self.exclude = exclude
self.options = options
self.keep = keep
self._last_backup = ''
self._output_dir = ''
self._inputs_path = ''
self._exclude_path = ''
self._err_flag = False
def check_params(self):
if self.inputs is None or len(self.inputs) == 0:
logger.info('No files or directory specified for backup.')
return False
for i in self.inputs:
if os.path.islink(i):
try:
i_new = os.readlink(i)
logger.info(f'Input {i} is a symbolic link referencing {i_new}. Copying {i_new} instead')
self.inputs.remove(i)
self.inputs.append(i_new)
except Exception:
logger.warning(f'Input {i} is a link and cannot be read. Skipping')
self.inputs.remove(i)
if not os.path.isdir(self.output):
logger.critical('Output path for backup does not exist')
return False
if self.keep is None:
self.keep = -1
return True
# Function to create the actual backup directory
def create_backup_dir(self):
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self._output_dir = f'{self.output}/simple_backup/{now}'
os.makedirs(self._output_dir, exist_ok=True)
def remove_old_backups(self):
try:
dirs = os.listdir(f'{self.output}/simple_backup')
except Exception:
logger.info('No older backups to remove')
return
if dirs.count('last_backup') > 0:
dirs.remove('last_backup')
n_backup = len(dirs) - 1
count = 0
if n_backup > self.keep:
logger.info('Removing old backups...')
dirs.sort()
for i in range(n_backup - self.keep):
try:
rmtree(f'{self.output}/simple_backup/{dirs[i]}')
count += 1
except Exception:
logger.error(f'Error while removing backup {dirs[i]}')
if count == 1:
logger.info(f'Removed {count} backup')
else:
logger.info(f'Removed {count} backups')
def find_last_backup(self):
if os.path.islink(f'{self.output}/simple_backup/last_backup'):
try:
self._last_backup = os.readlink(f'{self.output}/simple_backup/last_backup')
except Exception:
logger.warning('Previous backup could not be read')
else:
logger.info('No previous backups available')
if not os.path.isdir(self._last_backup):
logger.warning('Previous backup could not be read')
# Function to read configuration file
@timing(logger)
def run(self):
logger.info('Starting backup...')
self.create_backup_dir()
self.find_last_backup()
if os.path.islink(f'{self.output}/simple_backup/last_backup'):
try:
os.remove(f'{self.output}/simple_backup/last_backup')
except Exception:
logger.error('Failed to remove last_backup link')
self._err_flag = True
_, self._inputs_path = mkstemp(prefix='tmp_inputs', text=True)
_, self._exclude_path = mkstemp(prefix='tmp_exclude', text=True)
with open(self._inputs_path, 'w') as fp:
for i in self.inputs:
if not os.path.exists(i):
logger.warning(f'Input {i} not found. Skipping')
else:
fp.write(i)
fp.write('\n')
with open(self._exclude_path, 'w') as fp:
for e in self.exclude:
fp.write(e)
fp.write('\n')
logger.info('Copying files. This may take a long time...')
if self._last_backup == '':
rsync = f'rsync {self.options} --exclude-from={self._exclude_path} ' +\
f'--files-from={self._inputs_path} / "{self._output_dir}" ' +\
'--ignore-missing-args'
else:
rsync = f'rsync {self.options} --link-dest="{self._last_backup}" --exclude-from=' +\
f'{self._exclude_path} --files-from={self._inputs_path} / "{self._output_dir}" ' +\
'--ignore-missing-args'
p = Popen(rsync, stdout=PIPE, stderr=STDOUT, shell=True)
output, _ = p.communicate()
output = output.decode("utf-8").split('\n')
logger.info(f'rsync: {output[-3]}')
logger.info(f'rsync: {output[-2]}')
try:
os.symlink(self._output_dir, f'{self.output}/simple_backup/last_backup')
except Exception:
logger.error('Failed to create last_backup link')
self._err_flag = True
if self.keep != -1:
self.remove_old_backups()
os.remove(self._inputs_path)
os.remove(self._exclude_path)
logger.info('Backup completed')
if self._err_flag:
logger.warning('Some errors occurred (check log for details)')
return 1
return 0
def _parse_arguments():
parser = argparse.ArgumentParser(prog='simple_backup',
description='A simple backup script written in Python that uses rsync to copy files',
epilog='Report bugs to dfucini<at>gmail<dot>com',
formatter_class=MyFormatter)
parser.add_argument('-c', '--config', default=f'{homedir}/.config/simple_backup/simple_backup.config',
help='Specify location of configuration file')
parser.add_argument('-i', '--input', nargs='+', help='Paths/files to backup')
parser.add_argument('-o', '--output', help='Output directory for the backup')
parser.add_argument('-e', '--exclude', nargs='+', help='Files/directories/patterns to exclude from the backup')
parser.add_argument('-k', '--keep', type=int, help='Number of old backups to keep')
parser.add_argument('-s', '--checksum', action='store_true',
help='Use checksum rsync option to compare files (MUCH SLOWER)')
args = parser.parse_args()
return args
def _read_config(config_file):
if not os.path.isfile(config_file):
logger.warning(f'Config file {config_file} does not exist')
return None, None, None, None
config = configparser.ConfigParser()
config.read(config_file)
inputs = config.get('default', 'inputs')
inputs = inputs.split(',')
output = config.get('default', 'backup_dir')
exclude = config.get('default', 'exclude')
exclude = exclude.split(',')
keep = config.getint('default', 'keep')
return inputs, output, exclude, keep
def simple_backup():
args = _parse_arguments()
inputs, output, exclude, keep = _read_config(args.config)
if args.input is not None:
inputs = args.input
if args.output is not None:
output = args.output
if args.exclude is not None:
exclude = args.exclude
if args.keep is not None:
keep = args.keep
if args.checksum:
backup_options = '-arcvh -H -X'
else:
backup_options = '-arvh -H -X'
output = os.path.abspath(output)
backup = Backup(inputs, output, exclude, keep, backup_options)
if backup.check_params():
try:
obj = dbus.SessionBus().get_object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
obj = dbus.Interface(obj, "org.freedesktop.Notifications")
obj.Notify("simple_backup", 0, "", "Starting backup...", "", [], {"urgency": 1}, 10000)
except dbus.exceptions.DBusException:
pass
status = backup.run()
try:
if status == 0:
obj.Notify("simple_backup", 0, "", "Backup finished.", "", [], {"urgency": 1}, 10000)
else:
obj.Notify("simple_backup", 0, "", "Backup finished. Some errors occurred.", "", [], {"urgency": 1}, 10000)
except NameError:
pass
if __name__ == '__main__':
simple_backup()

View File

@ -0,0 +1,3 @@
"""Init."""
__version__ = '4.0.0'

View File

@ -0,0 +1,22 @@
# Example config file for simple_backup
[backup]
# Files and directories to backup. Multiple items can be separated using a comma (','). It is possible to use wildcards (i.e. '*' to match multiple characters and '~' for the user's home directory).
inputs=/home/my_home,/etc
# Output directory.
backup_dir=/media/Backup
# Files, directories and patterns to exclude from the backup. Multiple items can be separated using a comma.
exclude=*.bak
# Number of old backups (i.e. excluding the one that's being created) to keep (use -1 to keep all)
keep=-1
# Uncomment the following section to enable backup to remote server through ssh
# [server]
# host=
# username=
# ssh_keyfile=
# remote_sudo=
# numeric_ids=

754
simple_backup/simple_backup.py Executable file
View File

@ -0,0 +1,754 @@
#!/usr/bin/env python3
"""
A simple python script that calls rsync to perform a backup
Parameters can be specified on the command line or using a configuration file
Backup to a remote server is also supported (experimental)
Classes:
MyFormatter
Backup
"""
# Import libraries
import sys
import os
import warnings
from functools import wraps
from shutil import rmtree, which
import shlex
import argparse
import configparser
import logging
from logging import StreamHandler
from timeit import default_timer
from subprocess import Popen, PIPE, STDOUT
from datetime import datetime
from tempfile import mkstemp
from getpass import getpass
from glob import glob
from dotenv import load_dotenv
import paramiko
from paramiko import RSAKey, Ed25519Key, ECDSAKey, DSSKey
warnings.filterwarnings('error')
try:
from systemd import journal
except ImportError:
journal = None
try:
import dbus
except ImportError:
pass
load_dotenv()
euid = os.geteuid()
if euid == 0:
user = os.getenv('SUDO_USER')
homedir = os.path.expanduser(f'~{user}')
else:
user = os.getenv('USER')
homedir = os.getenv('HOME')
logging.getLogger().setLevel(logging.DEBUG)
logger = logging.getLogger(os.path.basename(__file__))
c_handler = StreamHandler()
c_handler.setLevel(logging.INFO)
c_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
c_handler.setFormatter(c_format)
logger.addHandler(c_handler)
if journal:
j_handler = journal.JournalHandler()
j_handler.setLevel(logging.INFO)
j_format = logging.Formatter('%(levelname)s - %(message)s')
j_handler.setFormatter(j_format)
logger.addHandler(j_handler)
def timing(_logger):
"""Decorator to measure execution time of a function
Parameters:
_logger: Logger object
"""
def decorator_timing(func):
@wraps(func)
def wrapper_timing(*args, **kwargs):
start = default_timer()
value = func(*args, **kwargs)
end = default_timer()
_logger.info(f'Elapsed time: {end - start:.3f} seconds')
return value
return wrapper_timing
return decorator_timing
class MyFormatter(argparse.RawTextHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
"""Custom format for argparse help text"""
class Backup:
"""Main class defining parameters and functions for performing backup
Attributes:
inputs: list
Files and folders that will be backup up
output: str
Path where the backup will be saved
exclude: list
List of files/folders/patterns to exclude from backup
options: str
String representing main backup options for rsync
keep: int
Number of old backup to preserve
host: str
Hostname of server (for remote backup)
username: str
Username for server login (for remote backup)
ssh_keyfile: str
Location of ssh key
remote_sudo: bool
Run remote rsync with sudo
remove_before: bool
Indicate if removing old backups will be performed before copying files
Methods:
check_params():
Check if parameters for the backup are valid
define_backup_dir():
Define the actual backup dir
remove_old_backups():
Remove old backups if there are more than indicated by 'keep'
find_last_backup():
Get path of last backup (from last_backup symlink) for rsync --link-dest
run():
Perform the backup
"""
def __init__(self, inputs, output, exclude, keep, options, host=None, username=None,
ssh_keyfile=None, remote_sudo=False, remove_before=False):
self.inputs = inputs
self.output = output
self.exclude = exclude
self.options = options
self.keep = keep
self.host = host
self.username = username
self.ssh_keyfile = ssh_keyfile
self.remote_sudo = remote_sudo
self._remove_before = remove_before
self._last_backup = ''
self._server = ''
self._output_dir = ''
self._inputs_path = ''
self._exclude_path = ''
self._remote = None
self._err_flag = False
self._ssh = None
self._password_auth = False
self._password = None
def check_params(self):
"""Check if parameters for the backup are valid"""
if self.inputs is None or len(self.inputs) == 0:
logger.info('No existing files or directories specified for backup. Nothing to do')
return 1
if self.output is None:
logger.critical('No output path specified. Use -o argument or specify output path in configuration file')
return 2
if self.host is not None and self.username is not None:
self._remote = True
if self._remote:
self._ssh = self._ssh_connect()
if self._ssh is None:
sys.exit(1)
_, stdout, _ = self._ssh.exec_command(f'if [ -d "{self.output}" ]; then echo "ok"; fi')
output = stdout.read().decode('utf-8').strip()
if output != 'ok':
logger.critical('Output path for backup does not exist')
return 2
else:
if not os.path.isdir(self.output):
logger.critical('Output path for backup does not exist')
return 2
self.output = os.path.abspath(self.output)
if self.keep is None:
self.keep = -1
return 0
# Function to create the actual backup directory
def define_backup_dir(self):
"""Define the actual backup dir"""
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self._output_dir = f'{self.output}/simple_backup/{now}'
if self._remote:
self._server = f'{self.username}@{self.host}:'
def remove_old_backups(self):
"""Remove old backups if there are more than indicated by 'keep'"""
if self._remote:
_, stdout, _ = self._ssh.exec_command(f'ls {self.output}/simple_backup')
dirs = stdout.read().decode('utf-8').strip().split('\n')
n_backup = len(dirs)
if not self._remove_before:
n_backup -= 1
count = 0
if n_backup > self.keep:
logger.info('Removing old backups...')
dirs.sort()
for i in range(n_backup - self.keep):
_, _, stderr = self._ssh.exec_command(f'rm -r "{self.output}/simple_backup/{dirs[i]}"')
err = stderr.read().decode('utf-8').strip().split('\n')[0]
if err != '':
logger.error('Error while removing backup %s.', {dirs[i]})
logger.error(err)
else:
count += 1
else:
try:
dirs = os.listdir(f'{self.output}/simple_backup')
except FileNotFoundError:
return
n_backup = len(dirs)
if not self._remove_before:
n_backup -= 1
count = 0
if n_backup > self.keep:
logger.info('Removing old backups...')
dirs.sort()
for i in range(n_backup - self.keep):
try:
rmtree(f'{self.output}/simple_backup/{dirs[i]}')
count += 1
except FileNotFoundError:
logger.error('Error while removing backup %s. Directory not found', dirs[i])
except PermissionError:
logger.error('Error while removing backup %s. Permission denied', dirs[i])
if count == 1:
logger.info('Removed %d backup', count)
elif count > 1:
logger.info('Removed %d backups', count)
def find_last_backup(self):
"""Get path of last backup (from last_backup symlink) for rsync --link-dest"""
if self._remote:
if self._ssh is None:
logger.critical('SSH connection to server failed')
sys.exit(1)
_, stdout, _ = self._ssh.exec_command(f'find {self.output}/simple_backup/ -mindepth 1 -maxdepth 1 -type d | sort')
output = stdout.read().decode('utf-8').strip().split('\n')
if output[-1] != '':
self._last_backup = output[-1]
else:
logger.info('No previous backups available')
else:
try:
dirs = sorted([f.path for f in os.scandir(f'{self.output}/simple_backup') if f.is_dir(follow_symlinks=False)])
except FileNotFoundError:
logger.info('No previous backups available')
return
except PermissionError:
logger.critical('Cannot access the backup directory. Permission denied')
try:
notify('Backup failed (check log for details)')
except NameError:
pass
sys.exit(3)
try:
self._last_backup = dirs[-1]
except IndexError:
logger.info('No previous backups available')
def _ssh_connect(self):
ssh = paramiko.SSHClient()
try:
ssh.load_host_keys(filename=f'{homedir}/.ssh/known_hosts')
except FileNotFoundError:
logger.warning(f'Cannot find file {homedir}/.ssh/known_hosts')
ssh.set_missing_host_key_policy(paramiko.WarningPolicy())
try:
ssh.connect(self.host, username=self.username)
return ssh
except UserWarning:
k = input(f'Unknown key for host {self.host}. Continue anyway? (Y/N) ')
if k[0].upper() == 'Y':
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
else:
return None
except paramiko.BadHostKeyException as e:
logger.critical('Can\'t connect to the server.')
logger.critical(e)
return None
except paramiko.SSHException:
pass
try:
ssh.connect(self.host, username=self.username)
return ssh
except paramiko.SSHException:
pass
if self.ssh_keyfile is None:
try:
password = getpass(f'{self.username}@{self.host}\'s password: ')
ssh.connect(self.host, username=self.username, password=password)
self._password_auth = True
os.environ['SSHPASS'] = password
return ssh
except paramiko.SSHException as e:
logger.critical('Can\'t connect to the server.')
logger.critical(e)
return None
pkey = None
try:
pkey = RSAKey.from_private_key_file(self.ssh_keyfile)
except paramiko.PasswordRequiredException:
password = getpass(f'Enter passwphrase for key \'{self.ssh_keyfile}\': ')
try:
pkey = RSAKey.from_private_key_file(self.ssh_keyfile, password)
except paramiko.SSHException:
pass
if pkey is None:
try:
pkey = Ed25519Key.from_private_key_file(self.ssh_keyfile)
except paramiko.PasswordRequiredException:
try:
pkey = Ed25519Key.from_private_key_file(self.ssh_keyfile, password)
except paramiko.SSHException:
pass
if pkey is None:
try:
pkey = ECDSAKey.from_private_key_file(self.ssh_keyfile)
except paramiko.PasswordRequiredException:
try:
pkey = ECDSAKey.from_private_key_file(self.ssh_keyfile, password)
except paramiko.SSHException:
pass
if pkey is None:
try:
pkey = DSSKey.from_private_key_file(self.ssh_keyfile)
except paramiko.PasswordRequiredException:
try:
pkey = DSSKey.from_private_key_file(self.ssh_keyfile, password)
except paramiko.SSHException:
pass
try:
ssh.connect(self.host, username=self.username, pkey=pkey)
except paramiko.SSHException:
logger.critical('SSH connection to server failed')
return None
return ssh
# Function to read configuration file
@timing(logger)
def run(self):
"""Perform the backup"""
logger.info('Starting backup...')
try:
_notify('Starting backup...')
except NameError:
pass
self.define_backup_dir()
self.find_last_backup()
_, self._inputs_path = mkstemp(prefix='tmp_inputs', text=True)
count = 0
with open(self._inputs_path, 'w', encoding='utf-8') as fp:
for i in self.inputs:
if not os.path.exists(i):
logger.warning('Input %s not found. Skipping', i)
else:
fp.write(i)
fp.write('\n')
count += 1
if count == 0:
logger.info('No existing files or directories specified for backup. Nothing to do')
try:
notify('Backup finished. No files copied')
except NameError:
pass
return 1
_, self._exclude_path = mkstemp(prefix='tmp_exclude', text=True)
with open(self._exclude_path, 'w', encoding='utf-8') as fp:
if self.exclude is not None:
for e in self.exclude:
fp.write(e)
fp.write('\n')
if self.keep != -1 and self._remove_before:
self.remove_old_backups()
logger.info('Copying files. This may take a long time...')
if self._last_backup == '':
rsync = f'/usr/bin/rsync {self.options} --exclude-from={self._exclude_path} ' +\
f'--files-from={self._inputs_path} / "{self._server}{self._output_dir}"'
else:
rsync = f'/usr/bin/rsync {self.options} --link-dest="{self._last_backup}" --exclude-from=' +\
f'{self._exclude_path} --files-from={self._inputs_path} / "{self._server}{self._output_dir}"'
if euid == 0 and self.ssh_keyfile is not None:
rsync = f'{rsync} -e \'ssh -i {self.ssh_keyfile} -o StrictHostKeyChecking=no\''
elif self._password_auth and which('sshpass'):
rsync = f'{rsync} -e \'sshpass -e ssh -l {self.username} -o StrictHostKeyChecking=no\''
else:
rsync = f'{rsync} -e \'ssh -o StrictHostKeyChecking=no\''
if self._remote and self.remote_sudo:
rsync = f'{rsync} --rsync-path="sudo rsync"'
args = shlex.split(rsync)
with Popen(args, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False) as p:
output, _ = p.communicate()
try:
del os.environ['SSHPASS']
except KeyError:
pass
if p.returncode != 0:
self._err_flag = True
output = output.decode("utf-8").split('\n')
if self._err_flag:
logger.error('rsync: %s', output)
else:
logger.info('rsync: %s', output[-3])
logger.info('rsync: %s', output[-2])
if self.keep != -1 and not self._remove_before:
self.remove_old_backups()
os.remove(self._inputs_path)
os.remove(self._exclude_path)
if self._remote:
_, stdout, _ = self._ssh.exec_command(f'if [ -d "{self._output_dir}" ]; then echo "ok"; fi')
output = stdout.read().decode('utf-8').strip()
if output == 'ok':
logger.info('Backup completed')
try:
_notify('Backup completed')
except NameError:
pass
else:
logger.error('Backup failed')
try:
_notify('Backup failed (check log for details)')
except NameError:
pass
if self._ssh:
self._ssh.close()
else:
if self._err_flag:
logger.error('Some errors occurred while performing the backup')
try:
_notify('Some errors occurred while performing the backup. Check log for details')
except NameError:
pass
return 4
else:
logger.info('Backup completed')
try:
_notify('Backup completed')
except NameError:
pass
return 0
def _parse_arguments():
parser = argparse.ArgumentParser(prog='simple_backup',
description='Simple backup script written in Python that uses rsync to copy files',
epilog='See simple_backup(1) manpage for full documentation',
formatter_class=MyFormatter)
parser.add_argument('-c', '--config', default=f'{homedir}/.config/simple_backup/simple_backup.conf',
help='Specify location of configuration file')
parser.add_argument('-i', '--inputs', nargs='+', help='Paths/files to backup')
parser.add_argument('-o', '--output', help='Output directory for the backup')
parser.add_argument('-e', '--exclude', nargs='+', help='Files/directories/patterns to exclude from the backup')
parser.add_argument('-k', '--keep', type=int, help='Number of old backups to keep')
parser.add_argument('--host', help='Server hostname (for remote backup)')
parser.add_argument('-u', '--username', help='Username to connect to server (for remote backup)')
parser.add_argument('--keyfile', help='SSH key location')
parser.add_argument('-s', '--checksum', action='store_true',
help='Use checksum rsync option to compare files')
parser.add_argument('-z', '--compress', action='store_true', help='Compress data during the transfer')
parser.add_argument('--remove-before-backup', action='store_true',
help='Remove old backups before executing the backup, instead of after')
parser.add_argument('--no-syslog', action='store_true', help='Disable systemd journal logging')
parser.add_argument('--rsync-options', nargs='+',
choices=['a', 'l', 'p', 't', 'g', 'o', 'c', 'h', 's', 'D', 'H', 'X'],
help='Specify options for rsync')
parser.add_argument('--remote-sudo', action='store_true', help='Run rsync on remote server with sudo if allowed')
parser.add_argument('--numeric-ids', action='store_true',
help='Use rsync \'--numeric-ids\' option (don\'t map uid/gid values by name')
args = parser.parse_args()
return args
def _expand_inputs(inputs):
expanded_inputs = []
for i in inputs:
if i == '':
continue
i_ex = glob(os.path.expanduser(i.replace('~', f'~{user}')))
if len(i_ex) == 0:
logger.warning('No file or directory matching input %s. Skipping...', i)
else:
expanded_inputs.extend(i_ex)
return expanded_inputs
def _read_config(config_file):
config_args = {}
if not os.path.isfile(config_file):
logger.warning('Config file %s does not exist', config_file)
return config_args
config = configparser.ConfigParser()
config.read(config_file)
section = 'backup'
# Allow compatibility with previous version of config file
try:
inputs = config.get(section, 'inputs')
except configparser.NoSectionError:
section = 'default'
inputs = config.get(section, 'inputs')
inputs = inputs.split(',')
inputs = _expand_inputs(inputs)
inputs = list(set(inputs))
config_args['inputs'] = inputs
output = config.get(section, 'backup_dir')
output = os.path.expanduser(output.replace('~', f'~{user}'))
config_args['output'] = output
try:
exclude = config.get(section, 'exclude')
exclude = exclude.split(',')
except configparser.NoOptionError:
exclude = []
config_args['exclude'] = exclude
try:
keep = config.getint(section, 'keep')
except configparser.NoOptionError:
keep = -1
config_args['keep'] = keep
try:
host = config.get('server', 'host')
username = config.get('server', 'username')
except (configparser.NoSectionError, configparser.NoOptionError):
host = None
username = None
config_args['host'] = host
config_args['username'] = username
try:
ssh_keyfile = config.get('server', 'ssh_keyfile')
except (configparser.NoSectionError, configparser.NoOptionError):
ssh_keyfile = None
config_args['ssh_keyfile'] = ssh_keyfile
try:
remote_sudo = config.getboolean('server', 'remote_sudo')
except (configparser.NoSectionError, configparser.NoOptionError):
remote_sudo = False
config_args['remote_sudo'] = remote_sudo
try:
numeric_ids = config.getboolean('server', 'numeric_ids')
except (configparser.NoSectionError, configparser.NoOptionError):
numeric_ids = False
config_args['numeric_ids'] = numeric_ids
return config_args
def _notify(text):
_euid = os.geteuid()
if _euid == 0:
uid = os.getenv('SUDO_UID')
else:
uid = os.geteuid()
os.seteuid(int(uid))
os.environ['DBUS_SESSION_BUS_ADDRESS'] = f'unix:path=/run/user/{uid}/bus'
obj = dbus.SessionBus().get_object('org.freedesktop.Notifications', '/org/freedesktop/Notifications')
obj = dbus.Interface(obj, 'org.freedesktop.Notifications')
obj.Notify('', 0, '', 'simple_backup', text, [], {'urgency': 1}, 10000)
os.seteuid(int(_euid))
def simple_backup():
"""Main"""
args = _parse_arguments()
if args.no_syslog:
try:
logger.removeHandler(j_handler)
except NameError:
pass
try:
config_args = _read_config(args.config)
except (configparser.NoSectionError, configparser.NoOptionError):
logger.critical('Bad configuration file')
sys.exit(1)
inputs = args.inputs if args.inputs is not None else config_args['inputs']
output = args.output if args.output is not None else config_args['output']
exclude = args.exclude if args.exclude is not None else config_args['exclude']
keep = args.keep if args.keep is not None else config_args['keep']
host = args.host if args.host is not None else config_args['host']
username = args.username if args.username is not None else config_args['username']
ssh_keyfile = args.keyfile if args.keyfile is not None else config_args['ssh_keyfile']
remote_sudo = args.remote_sudo if args.remote_sudo is not None else config_args['remote_sudo']
if args.rsync_options is None:
rsync_options = ['-a', '-r', '-v', '-h', '-H', '-X', '-s', '--ignore-missing-args', '--mkpath']
else:
rsync_options = ['-r', '-v']
for ro in args.rsync_options:
rsync_options.append(f'-{ro}')
if args.checksum:
rsync_options.append('-c')
if args.compress:
rsync_options.append('-z')
if args.numeric_ids or config_args['numeric_ids']:
rsync_options.append('--numeric-ids')
rsync_options = ' '.join(rsync_options)
backup = Backup(inputs, output, exclude, keep, rsync_options, host, username, ssh_keyfile,
remote_sudo, remove_before=args.remove_before_backup)
return_code = backup.check_params()
if return_code == 0:
return backup.run()
return return_code
if __name__ == '__main__':
simple_backup()