Compare commits

..
3 Commits
14 changed files with 236 additions and 64 deletions
+9 -2
View File
@@ -1,3 +1,10 @@
1.2.18
* #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
borgmatic's location section.
1.2.17
* #140: List the files within an archive via --list --archive option.
1.2.16
* #119: Include a sample borgmatic configuration file in the documentation.
* #123: Support for Borg archive restoration via borgmatic --extract command-line flag.
@@ -36,8 +43,8 @@
* #108: Support for Borg create --progress via borgmatic command-line flag.
1.2.10
* #105: Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage
section.
* #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
storage section.
1.2.9
* #102: Fix for syntax error that occurred in Python 3.5 and below.
+1
View File
@@ -141,6 +141,7 @@ def create_archive(
+ (('--compression', compression) if compression else ())
+ (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
+ (('--one-file-system',) if location_config.get('one_file_system') else ())
+ (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+ (('--read-special',) if location_config.get('read_special') else ())
+ (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
+ (('--files-cache', files_cache) if files_cache else ())
+4 -2
View File
@@ -57,6 +57,7 @@ def extract_archive(
repository,
archive,
restore_paths,
location_config,
storage_config,
local_path='borg',
remote_path=None,
@@ -64,8 +65,8 @@ def extract_archive(
):
'''
Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
restore from the archive, and a storage configuration dict, extract the archive into the current
directory.
restore from the archive, and location/storage configuration dicts, extract the archive into the
current directory.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
@@ -74,6 +75,7 @@ def extract_archive(
(local_path, 'extract', '::'.join((repository, archive)))
+ (tuple(restore_paths) if restore_paths else ())
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+ (('--umask', str(umask)) if umask else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+6 -4
View File
@@ -5,15 +5,17 @@ import subprocess
logger = logging.getLogger(__name__)
def list_archives(repository, storage_config, local_path='borg', remote_path=None, json=False):
def list_archives(
repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
):
'''
Given a local or remote repository path, and a storage config dict,
list Borg archives in the repository.
Given a local or remote repository path and a storage config dict, list Borg archives in the
repository. Or, if an archive name is given, list the files in that archive.
'''
lock_wait = storage_config.get('lock_wait', None)
full_command = (
(local_path, 'list', repository)
(local_path, 'list', '::'.join((repository, archive)) if archive else repository)
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+55 -32
View File
@@ -102,27 +102,49 @@ def parse_arguments(*arguments):
help='Create a repository with a fixed storage quota',
)
prune_group = parser.add_argument_group('options for --prune')
stats_argument = prune_group.add_argument(
'--stats',
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive',
)
create_group = parser.add_argument_group('options for --create')
create_group.add_argument(
progress_argument = create_group.add_argument(
'--progress',
dest='progress',
default=False,
action='store_true',
help='Display progress for each file as it is backed up',
help='Display progress for each file as it is processed',
)
create_group._group_actions.append(stats_argument)
json_argument = create_group.add_argument(
'--json', dest='json', default=False, action='store_true', help='Output results as JSON'
)
extract_group = parser.add_argument_group('options for --extract')
extract_group.add_argument(
repository_argument = extract_group.add_argument(
'--repository',
help='Path of repository to restore from, defaults to the configured repository if there is only one',
help='Path of repository to use, defaults to the configured repository if there is only one',
)
extract_group.add_argument('--archive', help='Name of archive to restore')
archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
extract_group.add_argument(
'--restore-path',
nargs='+',
dest='restore_paths',
help='Paths to restore from archive, defaults to the entire archive',
)
extract_group._group_actions.append(progress_argument)
list_group = parser.add_argument_group('options for --list')
list_group._group_actions.append(repository_argument)
list_group._group_actions.append(archive_argument)
list_group._group_actions.append(json_argument)
info_group = parser.add_argument_group('options for --info')
info_group._group_actions.append(json_argument)
common_group = parser.add_argument_group('common options')
common_group.add_argument(
@@ -140,20 +162,6 @@ def parse_arguments(*arguments):
dest='excludes_filename',
help='Deprecated in favor of exclude_patterns within configuration',
)
common_group.add_argument(
'--stats',
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive with --create or --prune option',
)
common_group.add_argument(
'--json',
dest='json',
default=False,
action='store_true',
help='Output results from the --create, --list, or --info options as json',
)
common_group.add_argument(
'-n',
'--dry-run',
@@ -196,10 +204,15 @@ def parse_arguments(*arguments):
raise ValueError('The --encryption option is required with the --init option')
if not args.extract:
if args.repository:
raise ValueError('The --repository option can only be used with the --extract option')
if args.archive:
raise ValueError('The --archive option can only be used with the --extract option')
if not args.list:
if args.repository:
raise ValueError(
'The --repository option can only be used with the --extract and --list options'
)
if args.archive:
raise ValueError(
'The --archive option can only be used with the --extract and --list options'
)
if args.restore_paths:
raise ValueError('The --restore-path option can only be used with the --extract option')
if args.extract and not args.archive:
@@ -354,20 +367,27 @@ def _run_commands_on_repository(
repository,
args.archive,
args.restore_paths,
location,
storage,
local_path=local_path,
remote_path=remote_path,
progress=args.progress,
)
if args.list:
logger.info('{}: Listing archives'.format(repository))
output = borg_list.list_archives(
repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
)
if args.json:
json_results.append(json.loads(output))
else:
sys.stdout.write(output)
if args.repository is None or repository == args.repository:
logger.info('{}: Listing archives'.format(repository))
output = borg_list.list_archives(
repository,
storage,
args.archive,
local_path=local_path,
remote_path=remote_path,
json=args.json,
)
if args.json:
json_results.append(json.loads(output))
else:
sys.stdout.write(output)
if args.info:
logger.info('{}: Displaying summary info for archives'.format(repository))
output = borg_info.display_archives_info(
@@ -388,6 +408,7 @@ def collect_configuration_run_summary_logs(config_filenames, args):
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict()
# Parse and load each configuration file.
for config_filename in config_filenames:
try:
logger.info('{}: Parsing configuration file'.format(config_filename))
@@ -403,13 +424,15 @@ def collect_configuration_run_summary_logs(config_filenames, args):
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if args.extract:
# Run cross-file validation checks.
if args.extract or (args.list and args.archive):
try:
validate.guard_configuration_contains_repository(args.repository, configs)
except ValueError as error:
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
return
# Execute the actions corresponding to each configuration file.
for config_filename, config in configs.items():
try:
run_configuration(config_filename, config, args)
+4
View File
@@ -32,6 +32,10 @@ map:
type: bool
desc: Stay in same file system (do not cross mount points). Defaults to false.
example: true
numeric_owner:
type: bool
desc: Only store/extract numeric user and group identifiers. Defaults to false.
example: true
read_special:
type: bool
desc: |
+1 -1
View File
@@ -130,7 +130,7 @@ def guard_configuration_contains_repository(repository, configurations):
if count > 1:
raise ValueError(
'Can\'t determine which repository to extract. Use --repository option to disambiguate'.format(
'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
repository
)
)
+9 -1
View File
@@ -45,9 +45,14 @@ not in your system `PATH`. Try looking in `/usr/local/bin/`.
This generates a sample configuration file at /etc/borgmatic/config.yaml (by
default). You should edit the file to suit your needs, as the values are
representative. All fields are optional except where indicated, so feel free
representative. All options are optional except where indicated, so feel free
to ignore anything you don't need.
Note that the configuration file is organized into distinct sections, each
with a section name like `location:` or `storage:`. So take care that if you
uncomment a particular option, also uncomment its containing section name, or
else borgmatic won't recognize the option.
You can also get the same sample configuration file from the [configuration
reference](../../docs/reference/configuration.md), the authoritative set of
all configuration options. This is handy if borgmatic has added new options
@@ -71,6 +76,9 @@ FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-e
for more info.
###
## Initialization
Before you can create backups with borgmatic, you first need to initialize a
+1 -1
View File
@@ -1,7 +1,7 @@
from setuptools import setup, find_packages
VERSION = '1.2.16'
VERSION = '1.2.18'
setup(
+44 -2
View File
@@ -142,14 +142,28 @@ def test_parse_arguments_disallows_init_and_dry_run():
)
def test_parse_arguments_disallows_repository_without_extract():
def test_parse_arguments_disallows_repository_without_extract_or_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
def test_parse_arguments_disallows_archive_without_extract():
def test_parse_arguments_allows_repository_with_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments(
'--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
)
def test_parse_arguments_allows_repository_with_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
def test_parse_arguments_disallows_archive_without_extract_or_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
@@ -169,6 +183,12 @@ def test_parse_arguments_allows_archive_with_extract():
module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
def test_parse_arguments_allows_archive_with_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
def test_parse_arguments_requires_archive_with_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
@@ -177,51 +197,73 @@ def test_parse_arguments_requires_archive_with_extract():
def test_parse_arguments_allows_progress_and_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--progress', '--create', '--list')
def test_parse_arguments_allows_progress_and_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
def test_parse_arguments_disallows_progress_without_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--progress', '--list')
def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats', '--create', '--list')
def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats', '--prune', '--list')
def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--stats', '--list')
def test_parse_arguments_with_just_stats_flag_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats')
def test_parse_arguments_allows_json_with_list_or_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--list', '--json')
module.parse_arguments('--info', '--json')
def test_parse_arguments_disallows_json_without_list_or_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--json')
def test_parse_arguments_disallows_json_with_both_list_and_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError):
module.parse_arguments('--list', '--info', '--json')
def test_borgmatic_version_matches_news_version():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
news_version = open('NEWS').readline()
+22 -1
View File
@@ -425,7 +425,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
)
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -446,6 +446,27 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
)
def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--numeric-owner',))
module.create_archive(
dry_run=False,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'numeric_owner': True,
'exclude_patterns': None,
},
storage_config={},
)
def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
+36 -3
View File
@@ -111,6 +111,7 @@ def test_extract_archive_calls_borg_with_restore_path_parameters():
repository='repo',
archive='archive',
restore_paths=['path1', 'path2'],
location_config={},
storage_config={},
)
@@ -123,11 +124,25 @@ def test_extract_archive_calls_borg_with_remote_path_parameters():
repository='repo',
archive='archive',
restore_paths=None,
location_config={},
storage_config={},
remote_path='borg1',
)
def test_extract_archive_calls_borg_with_numeric_owner_parameter():
insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--numeric-owner'))
module.extract_archive(
dry_run=False,
repository='repo',
archive='archive',
restore_paths=None,
location_config={'numeric_owner': True},
storage_config={},
)
def test_extract_archive_calls_borg_with_umask_parameters():
insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--umask', '0770'))
@@ -136,6 +151,7 @@ def test_extract_archive_calls_borg_with_umask_parameters():
repository='repo',
archive='archive',
restore_paths=None,
location_config={},
storage_config={'umask': '0770'},
)
@@ -148,6 +164,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters():
repository='repo',
archive='archive',
restore_paths=None,
location_config={},
storage_config={'lock_wait': '5'},
)
@@ -157,7 +174,12 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
insert_logging_mock(logging.INFO)
module.extract_archive(
dry_run=False, repository='repo', archive='archive', restore_paths=None, storage_config={}
dry_run=False,
repository='repo',
archive='archive',
restore_paths=None,
location_config={},
storage_config={},
)
@@ -166,7 +188,12 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
insert_logging_mock(logging.DEBUG)
module.extract_archive(
dry_run=False, repository='repo', archive='archive', restore_paths=None, storage_config={}
dry_run=False,
repository='repo',
archive='archive',
restore_paths=None,
location_config={},
storage_config={},
)
@@ -174,7 +201,12 @@ def test_extract_archive_calls_borg_with_dry_run_parameter():
insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--dry-run'))
module.extract_archive(
dry_run=True, repository='repo', archive='archive', restore_paths=None, storage_config={}
dry_run=True,
repository='repo',
archive='archive',
restore_paths=None,
location_config={},
storage_config={},
)
@@ -186,6 +218,7 @@ def test_extract_archive_calls_borg_with_progress_parameter():
repository='repo',
archive='archive',
restore_paths=None,
location_config={},
storage_config={},
progress=True,
)
+14 -7
View File
@@ -34,10 +34,18 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
module.list_archives(repository='repo', storage_config={})
def test_list_archives_with_json_calls_borg_with_json_parameter():
insert_subprocess_mock(LIST_COMMAND + ('--json',))
def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config = {'lock_wait': 5}
insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
module.list_archives(repository='repo', storage_config={}, json=True)
module.list_archives(repository='repo', storage_config=storage_config)
def test_list_archives_with_archive_calls_borg_with_archive_parameter():
storage_config = {}
insert_subprocess_mock(('borg', 'list', 'repo::archive'))
module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
def test_list_archives_with_local_path_calls_borg_via_local_path():
@@ -52,8 +60,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config = {'lock_wait': 5}
insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
def test_list_archives_with_json_calls_borg_with_json_parameter():
insert_subprocess_mock(LIST_COMMAND + ('--json',))
module.list_archives(repository='repo', storage_config=storage_config)
module.list_archives(repository='repo', storage_config={}, json=True)
+30 -8
View File
@@ -50,22 +50,22 @@ def test_run_commands_handles_multiple_json_outputs_in_array():
def test_collect_configuration_run_summary_logs_info_for_success():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration')
args = flexmock(extract=False)
args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.INFO)
assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration')
args = flexmock(extract=True, repository='repo')
args = flexmock(extract=True, list=False, repository='repo')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.INFO)
assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
@@ -73,16 +73,38 @@ def test_collect_configuration_run_summary_logs_critical_for_extract_with_reposi
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError
)
args = flexmock(extract=True, repository='repo')
args = flexmock(extract=True, list=False, repository='repo')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError
)
args = flexmock(extract=False, list=True, repository='repo', archive='test')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
def test_collect_configuration_run_summary_logs_info_for_success_with_list():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration')
args = flexmock(extract=False, list=True, repository='repo', archive=None)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_critical_for_parse_error():
flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
args = flexmock(extract=False)
args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
@@ -93,7 +115,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration').and_raise(ValueError)
args = flexmock(extract=False)
args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
@@ -103,7 +125,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration')
args = flexmock(config_paths=(), extract=False)
args = flexmock(config_paths=(), extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args))