From 123f8958f63810dfad252748a3cdaf16bc584dcf Mon Sep 17 00:00:00 2001 From: Tobias Schlatter Date: Wed, 25 Jun 2025 16:24:07 +0000 Subject: [PATCH] Create a checkpoint archive first with streaming processes (#1032). This ensures that if a streaming process fails, we do not create a real (i.e. non-checkpoint archive), (fixes #1032). Without this, there is a race condition where borg already creates an archive before borgmatic can kill it. Tested locally (with `pipx install --editable .`): ```sh borgmatic --config config.yaml create borgmatic --config config.yaml create --json ``` ## Hold up! Thanks for your contribution. Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see: https://torsion.org/borgmatic/#contributing ... which provides full instructions on how to submit pull requests. You can even use your GitHub account to login. Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1102 Reviewed-by: Dan Helfman Co-authored-by: Tobias Schlatter Co-committed-by: Tobias Schlatter --- borgmatic/actions/create.py | 87 +++++++++++- borgmatic/borg/create.py | 8 +- borgmatic/borg/rename.py | 62 +++++++++ borgmatic/borg/repo_list.py | 40 +++++- tests/integration/borg/test_rename.py | 120 ++++++++++++++++ tests/unit/actions/test_create.py | 189 ++++++++++++++++++++++++++ tests/unit/borg/test_create.py | 28 ++++ tests/unit/borg/test_rename.py | 37 +++++ tests/unit/borg/test_repo_list.py | 133 ++++++++++-------- 9 files changed, 643 insertions(+), 61 deletions(-) create mode 100644 borgmatic/borg/rename.py create mode 100644 tests/integration/borg/test_rename.py create mode 100644 tests/unit/borg/test_rename.py diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 0c7c4634..7f342c92 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -2,6 +2,8 @@ import logging import borgmatic.actions.json import borgmatic.borg.create +import borgmatic.borg.rename +import borgmatic.borg.repo_list import borgmatic.config.paths import borgmatic.config.validate import borgmatic.hooks.dispatch @@ -72,6 +74,12 @@ def run_create( ) stream_processes = [process for processes in active_dumps.values() for process in processes] + # If we have stream processes, we first create an archive with .checkpoint suffix. + # This is to make sure we only create a real archive if all the + # streaming processes completed successfully (create_archive will fail + # if a streaming process fails, but the archive might have already been created at this point). + use_checkpoint = bool(stream_processes) + json_output = borgmatic.borg.create.create_archive( global_arguments.dry_run, repository['path'], @@ -80,14 +88,40 @@ def run_create( local_borg_version, global_arguments, borgmatic_runtime_directory, + archive_suffix='.checkpoint' if use_checkpoint else '', local_path=local_path, remote_path=remote_path, json=create_arguments.json, stream_processes=stream_processes, ) + if use_checkpoint: + rename_checkpoint_archive( + repository['path'], + global_arguments, + config, + local_borg_version, + local_path, + remote_path, + ) + if json_output: - yield borgmatic.actions.json.parse_json(json_output, repository.get('label')) + output = borgmatic.actions.json.parse_json(json_output, repository.get('label')) + if use_checkpoint: + # Patch archive name and ID + renamed_archive = borgmatic.borg.repo_list.get_latest_archive( + repository['path'], + config, + local_borg_version, + global_arguments, + local_path, + remote_path, + ) + + output['archive']['name'] = renamed_archive['name'] + output['archive']['id'] = renamed_archive['id'] + + yield output borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', @@ -96,3 +130,54 @@ def run_create( borgmatic_runtime_directory, global_arguments.dry_run, ) + + +def rename_checkpoint_archive( + repository_path, + global_arguments, + config, + local_borg_version, + local_path, + remote_path, +): + ''' + Renames the latest archive to not have a '.checkpoint' suffix. + + Raises ValueError if + - there is not latest archive + - the latest archive does not have a '.checkpoint' suffix + + Implementation note: We cannot reliably get the just created archive name. + So we resort to listing the archives and picking the last one. + + A similar comment applies to retrieving the ID of the renamed archive. + ''' + archive = borgmatic.borg.repo_list.get_latest_archive( + repository_path, + config, + local_borg_version, + global_arguments, + local_path, + remote_path, + consider_checkpoints=True, + ) + + archive_name = archive['name'] + + if not archive_name.endswith('.checkpoint'): + raise ValueError(f'Latest archive did not have a .checkpoint suffix. Got: {archive_name}') + + new_archive_name = archive_name.removesuffix('.checkpoint') + + logger.info(f'Renaming archive {archive_name} -> {new_archive_name}') + + borgmatic.borg.rename.rename_archive( + repository_path, + archive_name, + new_archive_name, + global_arguments.dry_run, + config, + local_borg_version, + local_path, + remote_path, + ) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 03524ff7..bf970e64 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -121,6 +121,7 @@ def make_base_create_command( local_borg_version, global_arguments, borgmatic_runtime_directory, + archive_suffix='', local_path='borg', remote_path=None, json=False, @@ -149,8 +150,9 @@ def make_base_create_command( lock_wait = config.get('lock_wait', None) list_filter_flags = flags.make_list_filter_flags(local_borg_version, dry_run) files_cache = config.get('files_cache') - archive_name_format = config.get( - 'archive_name_format', flags.get_default_archive_name_format(local_borg_version) + archive_name_format = ( + config.get('archive_name_format', flags.get_default_archive_name_format(local_borg_version)) + + archive_suffix ) extra_borg_options = config.get('extra_borg_options', {}).get('create', '') @@ -269,6 +271,7 @@ def create_archive( local_borg_version, global_arguments, borgmatic_runtime_directory, + archive_suffix='', local_path='borg', remote_path=None, json=False, @@ -294,6 +297,7 @@ def create_archive( local_borg_version, global_arguments, borgmatic_runtime_directory, + archive_suffix, local_path, remote_path, json, diff --git a/borgmatic/borg/rename.py b/borgmatic/borg/rename.py new file mode 100644 index 00000000..5700c5a9 --- /dev/null +++ b/borgmatic/borg/rename.py @@ -0,0 +1,62 @@ +import logging + +import borgmatic.borg.flags + +logger = logging.getLogger(__name__) + + +def make_rename_command( + dry_run, + repository_name, + old_archive_name, + new_archive_name, + config, + local_borg_version, + local_path, + remote_path, +): + return ( + (local_path, 'rename') + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + borgmatic.borg.flags.make_flags('dry-run', dry_run) + + borgmatic.borg.flags.make_flags('remote-path', remote_path) + + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + + borgmatic.borg.flags.make_flags('log-json', config.get('log_json')) + + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) + + borgmatic.borg.flags.make_repository_archive_flags( + repository_name, old_archive_name, local_borg_version + ) + + (new_archive_name,) + ) + + +def rename_archive( + repository_name, + old_archive_name, + new_archive_name, + dry_run, + config, + local_borg_version, + local_path, + remote_path, +): + command = make_rename_command( + dry_run, + repository_name, + old_archive_name, + new_archive_name, + config, + local_borg_version, + local_path, + remote_path, + ) + + borgmatic.execute.execute_command( + command, + output_log_level=logging.INFO, + environment=borgmatic.borg.environment.make_environment(config), + working_directory=borgmatic.config.paths.get_working_directory(config), + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index 73d8ed88..e2ec114b 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -1,4 +1,5 @@ import argparse +import json import logging import borgmatic.config.paths @@ -29,6 +30,33 @@ def resolve_archive_name( if archive != 'latest': return archive + latest_archive = get_latest_archive( + repository_path, + config, + local_borg_version, + global_arguments, + local_path=local_path, + remote_path=remote_path, + ) + + return latest_archive['name'] + + +def get_latest_archive( + repository_path, + config, + local_borg_version, + global_arguments, + local_path='borg', + remote_path=None, + consider_checkpoints=False, +): + ''' + Returns a dict with information about the latest archive of a repository. + + Raises ValueError if there are no archives in the repository. + ''' + full_command = ( ( local_path, @@ -42,24 +70,28 @@ def resolve_archive_name( + flags.make_flags('umask', config.get('umask')) + flags.make_flags('log-json', config.get('log_json')) + flags.make_flags('lock-wait', config.get('lock_wait')) + + flags.make_flags('consider-checkpoints', consider_checkpoints) + flags.make_flags('last', 1) - + ('--short',) + + ('--json',) + flags.make_repository_flags(repository_path, local_borg_version) ) - output = execute_command_and_capture_output( + json_output = execute_command_and_capture_output( full_command, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), ) + + archives = json.loads(json_output)['archives'] + try: - latest_archive = output.strip().splitlines()[-1] + latest_archive = archives[-1] except IndexError: raise ValueError('No archives found in the repository') - logger.debug(f'Latest archive is {latest_archive}') + logger.debug(f'Latest archive is {latest_archive["name"]}') return latest_archive diff --git a/tests/integration/borg/test_rename.py b/tests/integration/borg/test_rename.py new file mode 100644 index 00000000..7269bfe6 --- /dev/null +++ b/tests/integration/borg/test_rename.py @@ -0,0 +1,120 @@ +import logging + +from borgmatic.borg import rename as module +from tests.unit.test_verbosity import insert_logging_mock + + +def test_make_rename_command_includes_log_info(): + insert_logging_mock(logging.INFO) + + command = module.make_rename_command( + dry_run=False, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={}, + local_borg_version='1.2.3', + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'rename', '--info', 'repo::old_archive', 'new_archive') + + +def test_make_rename_command_includes_log_debug(): + insert_logging_mock(logging.DEBUG) + + command = module.make_rename_command( + dry_run=False, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={}, + local_borg_version='1.2.3', + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'rename', '--debug', '--show-rc', 'repo::old_archive', 'new_archive') + + +def test_make_rename_command_includes_dry_run(): + command = module.make_rename_command( + dry_run=True, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={}, + local_borg_version='1.2.3', + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'rename', '--dry-run', 'repo::old_archive', 'new_archive') + + +def test_make_rename_command_includes_remote_path(): + command = module.make_rename_command( + dry_run=False, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={}, + local_borg_version='1.2.3', + local_path='borg', + remote_path='borg1', + ) + + assert command == ( + 'borg', + 'rename', + '--remote-path', + 'borg1', + 'repo::old_archive', + 'new_archive', + ) + + +def test_make_rename_command_includes_umask(): + command = module.make_rename_command( + dry_run=False, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={'umask': '077'}, + local_borg_version='1.2.3', + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'rename', '--umask', '077', 'repo::old_archive', 'new_archive') + + +def test_make_rename_command_includes_log_json(): + command = module.make_rename_command( + dry_run=False, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={'log_json': True}, + local_borg_version='1.2.3', + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'rename', '--log-json', 'repo::old_archive', 'new_archive') + + +def test_make_rename_command_includes_lock_wait(): + command = module.make_rename_command( + dry_run=False, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={'lock_wait': 5}, + local_borg_version='1.2.3', + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'rename', '--lock-wait', '5', 'repo::old_archive', 'new_archive') diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 722b6365..46de2510 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -1,3 +1,4 @@ +import json import os import pytest @@ -227,3 +228,191 @@ def test_run_create_produces_json(): remote_path=None, ) ) == [parsed_json] + + +def test_run_create_with_active_dumps_roundtrips_via_checkpoint_archive(): + mock_dump_process = flexmock() + mock_dump_process.should_receive('poll').and_return(None).and_return(0) + + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return( + {'dump': mock_dump_process} + ) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(()) + flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return([]) + flexmock(os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') + flexmock(module.borgmatic.borg.repo_list).should_receive('get_latest_archive').and_return( + {'id': 'id1', 'name': 'archive.checkpoint'} + ) + + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + flexmock(module).should_receive('rename_checkpoint_archive').with_args( + repository_path='repo', + global_arguments=global_arguments, + config={}, + local_borg_version=None, + local_path=None, + remote_path=None, + ).once() + create_arguments = flexmock( + repository=None, + progress=flexmock(), + statistics=flexmock(), + json=False, + list_details=flexmock(), + ) + + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_with_active_dumps_json_updates_archive_info(): + mock_dump_process = flexmock() + mock_dump_process.should_receive('poll').and_return(None).and_return(0) + + borg_create_result = { + 'archive': { + 'command_line': ['foo'], + 'name': 'archive.checkpoint', + 'id': 'id1', + }, + 'cache': {}, + 'repository': { + 'id': 'repo-id', + }, + } + + expected_create_result = { + 'archive': { + 'command_line': ['foo'], + 'name': 'archive', + 'id': 'id2', + }, + 'cache': {}, + 'repository': {'id': 'repo-id', 'label': ''}, + } + + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( + flexmock() + ) + + flexmock(module.borgmatic.borg.create).should_receive('create_archive').and_return( + json.dumps(borg_create_result) + ).once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return( + {'dump': mock_dump_process} + ) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(()) + flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return([]) + flexmock(os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') + + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + flexmock(module).should_receive('rename_checkpoint_archive').with_args( + repository_path='repo', + global_arguments=global_arguments, + config={}, + local_borg_version=None, + local_path=None, + remote_path=None, + ).once() + + flexmock(module.borgmatic.borg.repo_list).should_receive('get_latest_archive').and_return( + {'id': 'id2', 'name': 'archive'}, + ) + + create_arguments = flexmock( + repository=None, + progress=flexmock(), + statistics=flexmock(), + json=True, + list_details=flexmock(), + ) + + assert list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) == [expected_create_result] + + +def test_rename_checkpoint_archive_renames_archive(): + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + flexmock(module.borgmatic.borg.repo_list).should_receive('get_latest_archive').and_return( + {'id': 'id1', 'name': 'archive.checkpoint'} + ) + + flexmock(module.borgmatic.borg.rename).should_receive('rename_archive').with_args( + repository_name='path', + old_archive_name='archive.checkpoint', + new_archive_name='archive', + dry_run=False, + config={}, + local_borg_version=None, + local_path=None, + remote_path=None, + ) + + module.rename_checkpoint_archive( + repository_path='path', + global_arguments=global_arguments, + config={}, + local_borg_version=None, + local_path=None, + remote_path=None, + ) + + +def test_rename_checkpoint_archive_checks_suffix(): + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + flexmock(module.borgmatic.borg.repo_list).should_receive('get_latest_archive').and_return( + {'id': 'id1', 'name': 'unexpected-archive'} + ) + + with pytest.raises( + ValueError, + match='Latest archive did not have a .checkpoint suffix. Got: unexpected-archive', + ): + module.rename_checkpoint_archive( + repository_path='path', + global_arguments=global_arguments, + config={}, + local_borg_version=None, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 2df0beee..f4db34ff 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -786,6 +786,34 @@ def test_make_base_create_command_includes_repository_and_archive_name_format_wi assert not pattern_file +def test_make_base_create_command_includes_archive_suffix_in_borg_command(): + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.flags).should_receive('get_default_archive_name_format').and_return( + DEFAULT_ARCHIVE_NAME + ) + flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(()) + + (create_flags, create_positional_arguments, pattern_file) = module.make_base_create_command( + dry_run=False, + repository_path='repo', + config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + }, + patterns=[Pattern('foo'), Pattern('bar')], + local_borg_version='1.2.3', + global_arguments=flexmock(), + borgmatic_runtime_directory='/run/borgmatic', + archive_suffix='.checkpoint', + ) + + assert create_flags == ('borg', 'create') + assert create_positional_arguments == (f'repo::{DEFAULT_ARCHIVE_NAME}.checkpoint',) + assert not pattern_file + + def test_make_base_create_command_includes_extra_borg_options_in_borg_command(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) diff --git a/tests/unit/borg/test_rename.py b/tests/unit/borg/test_rename.py new file mode 100644 index 00000000..dc299328 --- /dev/null +++ b/tests/unit/borg/test_rename.py @@ -0,0 +1,37 @@ +import logging + +from flexmock import flexmock + +from borgmatic.borg import rename as module + + +def test_rename_archive_calls_borg_rename(): + environment = flexmock() + + # Note: make_rename_command is tested as integration test. + flexmock(module).should_receive('make_rename_command').and_return(('borg', 'fake-command')) + flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( + environment + ) + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( + '/working/dir' + ) + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + ('borg', 'fake-command'), + output_log_level=logging.INFO, + environment=environment, + working_directory='/working/dir', + borg_local_path='borg', + borg_exit_codes=None, + ).once() + + module.rename_archive( + dry_run=False, + repository_name='repo', + old_archive_name='old_archive', + new_archive_name='new_archive', + config={}, + local_borg_version='1.2.3', + local_path='borg', + remote_path=None, + ) diff --git a/tests/unit/borg/test_repo_list.py b/tests/unit/borg/test_repo_list.py index 4f3c1484..96a6b54d 100644 --- a/tests/unit/borg/test_repo_list.py +++ b/tests/unit/borg/test_repo_list.py @@ -1,4 +1,5 @@ import argparse +import json import logging import pytest @@ -11,7 +12,7 @@ from ..test_verbosity import insert_logging_mock BORG_LIST_LATEST_ARGUMENTS = ( '--last', '1', - '--short', + '--json', 'repo', ) @@ -31,8 +32,43 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name(): ) -def test_resolve_archive_name_calls_borg_with_flags(): +def test_resolve_archive_name_calls_get_latest_archive(): expected_archive = 'archive-name' + + repository_path = flexmock() + config = flexmock() + local_borg_version = flexmock() + global_arguments = flexmock() + local_path = flexmock() + remote_path = flexmock() + + flexmock(module.environment).should_receive('make_environment') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + flexmock(module).should_receive('get_latest_archive').with_args( + repository_path, + config, + local_borg_version, + global_arguments, + local_path, + remote_path, + ).and_return({'name': expected_archive}) + + assert ( + module.resolve_archive_name( + repository_path, + 'latest', + config, + local_borg_version, + global_arguments, + local_path, + remote_path, + ) + == expected_archive + ) + + +def test_get_latest_archive_calls_borg_with_flags(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -41,12 +77,11 @@ def test_resolve_archive_name_calls_borg_with_flags(): borg_exit_codes=None, environment=None, working_directory=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -55,8 +90,8 @@ def test_resolve_archive_name_calls_borg_with_flags(): ) -def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_log_info_calls_borg_without_info_flag(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -65,13 +100,12 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): working_directory=None, borg_local_path='borg', borg_exit_codes=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) insert_logging_mock(logging.INFO) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -80,8 +114,8 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): ) -def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_log_debug_calls_borg_without_debug_flag(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -90,13 +124,12 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): working_directory=None, borg_local_path='borg', borg_exit_codes=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) insert_logging_mock(logging.DEBUG) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -105,8 +138,8 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): ) -def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_local_path_calls_borg_via_local_path(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -115,12 +148,11 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): working_directory=None, borg_local_path='borg1', borg_exit_codes=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -130,8 +162,8 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): ) -def test_resolve_archive_name_with_exit_codes_calls_borg_using_them(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_exit_codes_calls_borg_using_them(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) borg_exit_codes = flexmock() @@ -141,12 +173,11 @@ def test_resolve_archive_name_with_exit_codes_calls_borg_using_them(): working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={'borg_exit_codes': borg_exit_codes}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -155,8 +186,8 @@ def test_resolve_archive_name_with_exit_codes_calls_borg_using_them(): ) -def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_remote_path_calls_borg_with_remote_path_flags(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -165,12 +196,11 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags working_directory=None, borg_local_path='borg', borg_exit_codes=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -180,8 +210,8 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags ) -def test_resolve_archive_name_with_umask_calls_borg_with_umask_flags(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_umask_calls_borg_with_umask_flags(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -190,12 +220,11 @@ def test_resolve_archive_name_with_umask_calls_borg_with_umask_flags(): working_directory=None, borg_local_path='borg', borg_exit_codes=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={'umask': '077'}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -204,7 +233,7 @@ def test_resolve_archive_name_with_umask_calls_borg_with_umask_flags(): ) -def test_resolve_archive_name_without_archives_raises(): +def test_get_latest_archive_without_archives_raises(): flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -213,20 +242,19 @@ def test_resolve_archive_name_without_archives_raises(): working_directory=None, borg_local_path='borg', borg_exit_codes=None, - ).and_return('') + ).and_return(json.dumps({'archives': []})) with pytest.raises(ValueError): - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={}, local_borg_version='1.2.3', global_arguments=flexmock(), ) -def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_log_json_calls_borg_with_log_json_flags(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) @@ -236,12 +264,11 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): working_directory=None, borg_local_path='borg', borg_exit_codes=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={'log_json': True}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -250,8 +277,8 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): ) -def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): - expected_archive = 'archive-name' +def test_get_latest_archive_with_lock_wait_calls_borg_with_lock_wait_flags(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) @@ -261,12 +288,11 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): working_directory=None, borg_local_path='borg', borg_exit_codes=None, - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={'lock_wait': 'okay'}, local_borg_version='1.2.3', global_arguments=flexmock(), @@ -275,8 +301,8 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): ) -def test_resolve_archive_name_calls_borg_with_working_directory(): - expected_archive = 'archive-name' +def test_get_latest_archive_calls_borg_with_working_directory(): + expected_archive = {'name': 'archive-name'} flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working/dir', @@ -287,12 +313,11 @@ def test_resolve_archive_name_calls_borg_with_working_directory(): borg_exit_codes=None, environment=None, working_directory='/working/dir', - ).and_return(expected_archive + '\n') + ).and_return(json.dumps({'archives': [expected_archive]})) assert ( - module.resolve_archive_name( + module.get_latest_archive( 'repo', - 'latest', config={'working_directory': '/working/dir'}, local_borg_version='1.2.3', global_arguments=flexmock(),