Compare commits

..
11 Commits
13 changed files with 227 additions and 78 deletions
+12
View File
@@ -1,3 +1,15 @@
1.2.14
* #103: When generating sample configuration with generate-borgmatic-config, document the defaults
for each option.
* #116: When running multiple configuration files, attempt all configuration files even if one of
them errors. Log a summary of results at the end.
* Add borgmatic --version command-line flag to get the current installed version number.
1.2.13
* #100: Support for --stats command-line flag independent of --verbosity.
* #117: With borgmatic --init command-line flag, proceed without erroring if a repository already
exists.
1.2.12
* #110: Support for Borg repository initialization via borgmatic --init command-line flag.
* #111: Update Borg create --filter values so a dry run lists files to back up.
+4
View File
@@ -146,6 +146,10 @@ Also, optionally check out the [Borg Quick
Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) for more
background about repository initialization.
Note that borgmatic skips repository initialization if the repository already
exists. This supports use cases like ensuring a repository exists prior to
performing a backup.
If the repository is on a remote host, make sure that your local user has
key-based SSH access to the desired user account on the remote host.
+2 -1
View File
@@ -91,6 +91,7 @@ def create_archive(
local_path='borg',
remote_path=None,
progress=False,
stats=False,
json=False,
):
'''
@@ -135,7 +136,7 @@ def create_archive(
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--list', '--filter', 'AME-') if logger.isEnabledFor(logging.INFO) else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ())
+ (('--stats',) if not dry_run and (logger.isEnabledFor(logging.INFO) or stats) else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ (('--dry-run',) if dry_run else ())
+ (('--progress',) if progress else ())
+12 -4
View File
@@ -15,9 +15,17 @@ def initialize_repository(
):
'''
Given a local or remote repository path, a Borg encryption mode, whether the repository should
be append-only, and the storage quota to use, initialize the repository.
be append-only, and the storage quota to use, initialize the repository. If the repository
already exists, then log and skip initialization.
'''
full_command = (
info_command = (local_path, 'info', repository)
logger.debug(' '.join(info_command))
if subprocess.call(info_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0:
logger.info('Repository already exists. Skipping initialization.')
return
init_command = (
(local_path, 'init', repository)
+ (('--encryption', encryption_mode) if encryption_mode else ())
+ (('--append-only',) if append_only else ())
@@ -27,5 +35,5 @@ def initialize_repository(
+ (('--remote-path', remote_path) if remote_path else ())
)
logger.debug(' '.join(full_command))
subprocess.check_call(full_command)
logger.debug(' '.join(init_command))
subprocess.check_call(init_command)
+8 -1
View File
@@ -31,7 +31,13 @@ def _make_prune_flags(retention_config):
def prune_archives(
dry_run, repository, storage_config, retention_config, local_path='borg', remote_path=None
dry_run,
repository,
storage_config,
retention_config,
local_path='borg',
remote_path=None,
stats=False,
):
'''
Given dry-run flag, a local or remote repository path, a storage config dict, and a
@@ -51,6 +57,7 @@ def prune_archives(
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ (('--dry-run',) if dry_run else ())
+ (('--stats',) if stats else ())
)
logger.debug(' '.join(full_command))
+76 -25
View File
@@ -5,6 +5,8 @@ import os
from subprocess import CalledProcessError
import sys
import pkg_resources
from borgmatic.borg import (
check as borg_check,
create as borg_create,
@@ -29,7 +31,7 @@ LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
def parse_arguments(*arguments):
'''
Given command-line arguments with which this script was invoked, parse the arguments and return
them as an ArgumentParser instance.
them as an argparse.ArgumentParser instance.
'''
config_paths = collect.get_default_config_paths()
@@ -107,6 +109,13 @@ def parse_arguments(*arguments):
action='store_true',
help='Display progress with --create option for each file as it is backed up',
)
parser.add_argument(
'--stats',
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive with --create or --prune option',
)
parser.add_argument(
'--json',
dest='json',
@@ -129,6 +138,13 @@ def parse_arguments(*arguments):
default=0,
help='Display verbose progress (1 for some, 2 for lots)',
)
parser.add_argument(
'--version',
dest='version',
default=False,
action='store_true',
help='Display installed version number of borgmatic and exit',
)
args = parser.parse_args(arguments)
@@ -142,16 +158,17 @@ def parse_arguments(*arguments):
'The --encryption, --append-only, and --storage-quota options can only be used with the --init option'
)
if args.init and (args.prune or args.create or args.dry_run):
raise ValueError(
'The --init option cannot be used with the --prune, --create, or --dry-run options'
)
if args.init and args.dry_run:
raise ValueError('The --init option cannot be used with the --dry-run option')
if args.init and not args.encryption_mode:
raise ValueError('The --encryption option is required with the --init option')
if args.progress and not args.create:
raise ValueError('The --progress option can only be used with the --create option')
if args.stats and not (args.create or args.prune):
raise ValueError('The --stats option can only be used with the --create or --prune options')
if args.json and not (args.create or args.list or args.info):
raise ValueError(
'The --json option can only be used with the --create, --list, or --info options'
@@ -261,6 +278,7 @@ def _run_commands_on_repository(
retention,
local_path=local_path,
remote_path=remote_path,
stats=args.stats,
)
if args.create:
logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
@@ -272,6 +290,7 @@ def _run_commands_on_repository(
local_path=local_path,
remote_path=remote_path,
progress=args.progress,
stats=args.stats,
)
if args.check and checks.repository_enabled_for_checks(repository, consistency):
logger.info('{}: Running consistency checks'.format(repository))
@@ -298,25 +317,57 @@ def _run_commands_on_repository(
sys.stdout.write(output)
def main(): # pragma: no cover
try:
configure_signals()
args = parse_arguments(*sys.argv[1:])
logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
logger.debug('Ensuring legacy configuration is upgraded')
convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
if len(config_filenames) == 0:
raise ValueError(
'Error: No configuration files found in: {}'.format(' '.join(args.config_paths))
)
for config_filename in config_filenames:
def collect_configuration_run_summary_logs(config_filenames, args):
'''
Given a sequence of configuration filenames and parsed command-line arguments as an
argparse.ArgumentParser instance, run each configuration file and yield a series of
logging.LogRecord instances containing summary information about each run.
'''
for config_filename in config_filenames:
try:
run_configuration(config_filename, args)
except (ValueError, OSError, CalledProcessError) as error:
print(error, file=sys.stderr)
print(file=sys.stderr)
print('Need some help? https://torsion.org/borgmatic/#issues', file=sys.stderr)
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
msg='{}: Successfully ran configuration file'.format(config_filename),
)
)
except (ValueError, OSError, CalledProcessError) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error running configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if not config_filenames:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
)
)
def main(): # pragma: no cover
configure_signals()
args = parse_arguments(*sys.argv[1:])
logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
if args.version:
print(pkg_resources.require('borgmatic')[0].version)
sys.exit(0)
config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
logger.debug('Ensuring legacy configuration is upgraded')
convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
summary_logs = tuple(collect_configuration_run_summary_logs(config_filenames, args))
logger.info('\nsummary:')
[logger.handle(log) for log in summary_logs if log.levelno >= logger.getEffectiveLevel()]
if any(log.levelno == logging.CRITICAL for log in summary_logs):
logger.critical('\nNeed some help? https://torsion.org/borgmatic/#issues')
sys.exit(1)
+20 -16
View File
@@ -30,14 +30,14 @@ map:
- user@backupserver:sourcehostname.borg
one_file_system:
type: bool
desc: Stay in same file system (do not cross mount points).
desc: Stay in same file system (do not cross mount points). Defaults to false.
example: true
read_special:
type: bool
desc: |
Use Borg's --read-special flag to allow backup of block and other special
devices. Use with caution, as it will lead to problems if used when
backing up special devices such as /dev/zero.
backing up special devices such as /dev/zero. Defaults to false.
example: false
bsd_flags:
type: bool
@@ -48,7 +48,7 @@ map:
desc: |
Mode in which to operate the files cache. See
https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
details.
details. Defaults to "ctime,size,inode".
example: ctime,size,inode
local_path:
type: scalar
@@ -102,11 +102,13 @@ map:
type: bool
desc: |
Exclude directories that contain a CACHEDIR.TAG file. See
http://www.brynosaurus.com/cachedir/spec.html for details.
http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false.
example: true
exclude_if_present:
type: scalar
desc: Exclude directories that contain a file with the given filename.
desc: |
Exclude directories that contain a file with the given filename. Defaults to not
set.
example: .nobackup
storage:
desc: |
@@ -121,7 +123,7 @@ map:
The standard output of this command is used to unlock the encryption key. Only
use on repositories that were initialized with passcommand/repokey encryption.
Note that if both encryption_passcommand and encryption_passphrase are set,
then encryption_passphrase takes precedence.
then encryption_passphrase takes precedence. Defaults to not set.
example: "secret-tool lookup borg-repository repo-name"
encryption_passphrase:
type: scalar
@@ -129,7 +131,7 @@ map:
Passphrase to unlock the encryption key with. Only use on repositories that were
initialized with passphrase/repokey encryption. Quote the value if it contains
punctuation, so it parses correctly. And backslash any quote or backslash
literals as well.
literals as well. Defaults to not set.
example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
checkpoint_interval:
type: int
@@ -143,7 +145,7 @@ map:
desc: |
Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
for details.
for details. Defaults to "19,23,21,4095".
example: 19,23,21,4095
compression:
type: scalar
@@ -154,25 +156,27 @@ map:
example: lz4
remote_rate_limit:
type: int
desc: Remote network upload rate limit in kiBytes/second.
desc: Remote network upload rate limit in kiBytes/second. Defaults to unlimited.
example: 100
ssh_command:
type: scalar
desc: Command to use instead of just "ssh". This can be used to specify ssh options.
desc: |
Command to use instead of just "ssh". This can be used to specify ssh options.
Defaults to not set.
example: ssh -i /path/to/private/key
umask:
type: scalar
desc: Umask to be used for borg create.
desc: Umask to be used for borg create. Defaults to 0077.
example: 0077
lock_wait:
type: int
desc: Maximum seconds to wait for acquiring a repository/cache lock.
desc: Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1.
example: 5
archive_name_format:
type: scalar
desc: |
Name of the archive. Borg placeholders can be used. See the output of
"borg help placeholders" for details. Default is
"borg help placeholders" for details. Defaults to
"{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
also specify a prefix in the retention section to avoid accidental pruning of
archives with a different archive name format. And you should also specify a
@@ -221,7 +225,7 @@ map:
desc: |
When pruning, only consider archive names starting with this prefix.
Borg placeholders can be used. See the output of "borg help placeholders" for
details. Default is "{hostname}-".
details. Defaults to "{hostname}-".
example: sourcehostname
consistency:
desc: |
@@ -256,14 +260,14 @@ map:
check_last:
type: int
desc: Restrict the number of checked archives to the last n. Applies only to the
"archives" check.
"archives" check. Defaults to checking all archives.
example: 3
prefix:
type: scalar
desc: |
When performing the "archives" check, only consider archive names starting with
this prefix. Borg placeholders can be used. See the output of
"borg help placeholders" for details. Default is "{hostname}-".
"borg help placeholders" for details. Defaults to "{hostname}-".
example: sourcehostname
hooks:
desc: |
+1
View File
@@ -41,6 +41,7 @@ for sub_command in prune create check list info; do
| grep -v '^--nobsdflags$' \
| grep -v '^--pattern$' \
| grep -v '^--progress$' \
| grep -v '^--stats$' \
| grep -v '^--read-special$' \
| grep -v '^--repository-only$' \
| grep -v '^--show-rc$' \
+1 -1
View File
@@ -1,7 +1,7 @@
from setuptools import setup, find_packages
VERSION = '1.2.12'
VERSION = '1.2.14'
setup(
+29 -7
View File
@@ -1,3 +1,5 @@
import subprocess
from flexmock import flexmock
import pytest
@@ -113,25 +115,25 @@ def test_parse_arguments_disallows_storage_quota_without_init():
module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
def test_parse_arguments_disallows_init_and_prune():
def test_parse_arguments_allows_init_and_prune():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--config', 'myconfig', '--init', '--prune')
module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--prune')
def test_parse_arguments_disallows_init_and_create():
def test_parse_arguments_allows_init_and_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--config', 'myconfig', '--init', '--create')
module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--create')
def test_parse_arguments_disallows_init_and_dry_run():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--config', 'myconfig', '--init', '--dry-run')
module.parse_arguments(
'--config', 'myconfig', '--init', '--encryption', 'repokey', '--dry-run'
)
def test_parse_arguments_allows_progress_and_create():
@@ -143,6 +145,19 @@ def test_parse_arguments_disallows_progress_without_create():
module.parse_arguments('--progress', '--list')
def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
module.parse_arguments('--stats', '--create', '--list')
def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
module.parse_arguments('--stats', '--prune', '--list')
def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
with pytest.raises(ValueError):
module.parse_arguments('--stats', '--list')
def test_parse_arguments_allows_json_with_list_or_info():
module.parse_arguments('--list', '--json')
module.parse_arguments('--info', '--json')
@@ -156,3 +171,10 @@ def test_parse_arguments_disallows_json_without_list_or_info():
def test_parse_arguments_disallows_json_with_both_list_and_info():
with pytest.raises(ValueError):
module.parse_arguments('--list', '--info', '--json')
def test_borgmatic_version_matches_news_version():
borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
news_version = open('NEWS').readline()
assert borgmatic_version == news_version
-8
View File
@@ -1,8 +0,0 @@
import subprocess
def test_setup_version_matches_news_version():
setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii')
news_version = open('NEWS').readline()
assert setup_version == news_version
+33 -12
View File
@@ -6,53 +6,74 @@ from borgmatic.borg import init as module
from ..test_verbosity import insert_logging_mock
def insert_subprocess_mock(check_call_command, **kwargs):
subprocess = flexmock(module.subprocess)
subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
INFO_REPOSITORY_EXISTS_RESPONSE_CODE = 0
INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE = 2
INIT_COMMAND = ('borg', 'init', 'repo', '--encryption', 'repokey')
def insert_info_command_mock(info_response):
subprocess = flexmock(module.subprocess)
subprocess.should_receive('call').and_return(info_response)
def insert_init_command_mock(init_command, **kwargs):
subprocess = flexmock(module.subprocess)
subprocess.should_receive('check_call').with_args(init_command, **kwargs).once()
def test_initialize_repository_calls_borg_with_parameters():
insert_subprocess_mock(INIT_COMMAND)
insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
insert_init_command_mock(INIT_COMMAND)
module.initialize_repository(repository='repo', encryption_mode='repokey')
def test_initialize_repository_skips_initialization_when_repository_already_exists():
insert_info_command_mock(INFO_REPOSITORY_EXISTS_RESPONSE_CODE)
flexmock(module.subprocess).should_receive('check_call').never()
module.initialize_repository(repository='repo', encryption_mode='repokey')
def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter():
insert_subprocess_mock(INIT_COMMAND + ('--append-only',))
insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
insert_init_command_mock(INIT_COMMAND + ('--append-only',))
module.initialize_repository(repository='repo', encryption_mode='repokey', append_only=True)
def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter():
insert_subprocess_mock(INIT_COMMAND + ('--storage-quota', '5G'))
insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G'))
module.initialize_repository(repository='repo', encryption_mode='repokey', storage_quota='5G')
def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
insert_subprocess_mock(INIT_COMMAND + ('--info',))
insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
insert_init_command_mock(INIT_COMMAND + ('--info',))
insert_logging_mock(logging.INFO)
module.initialize_repository(repository='repo', encryption_mode='repokey')
def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
insert_subprocess_mock(INIT_COMMAND + ('--debug',))
insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
insert_init_command_mock(INIT_COMMAND + ('--debug',))
insert_logging_mock(logging.DEBUG)
module.initialize_repository(repository='repo', encryption_mode='repokey')
def test_initialize_repository_with_local_path_calls_borg_via_local_path():
insert_subprocess_mock(('borg1',) + INIT_COMMAND[1:])
insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
insert_init_command_mock(('borg1',) + INIT_COMMAND[1:])
module.initialize_repository(repository='repo', encryption_mode='repokey', local_path='borg1')
def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter():
insert_subprocess_mock(INIT_COMMAND + ('--remote-path', 'borg1'))
insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1'))
module.initialize_repository(repository='repo', encryption_mode='repokey', remote_path='borg1')
+29 -3
View File
@@ -3,12 +3,12 @@ import sys
from flexmock import flexmock
from borgmatic.commands import borgmatic
from borgmatic.commands import borgmatic as module
def test_run_commands_handles_multiple_json_outputs_in_array():
(
flexmock(borgmatic)
flexmock(module)
.should_receive('_run_commands_on_repository')
.times(3)
.replace_with(
@@ -36,7 +36,7 @@ def test_run_commands_handles_multiple_json_outputs_in_array():
)
)
borgmatic._run_commands(
module._run_commands(
args=flexmock(json=True),
consistency=None,
local_path=None,
@@ -45,3 +45,29 @@ def test_run_commands_handles_multiple_json_outputs_in_array():
retention=None,
storage=None,
)
def test_collect_configuration_run_summary_logs_info_for_success():
flexmock(module).should_receive('run_configuration')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=()))
assert any(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_critical_for_error():
flexmock(module).should_receive('run_configuration').and_raise(ValueError)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=()))
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
logs = tuple(
module.collect_configuration_run_summary_logs(
config_filenames=(), args=flexmock(config_paths=())
)
)
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)