diff --git a/NEWS b/NEWS index 12c1f957..f3a34e9b 100644 --- a/NEWS +++ b/NEWS @@ -8,6 +8,13 @@ * #1228: Adjust the "spot" check so error output includes more information about what failed. * #1236: Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them. * #1243: Add a "diff" action for viewing the difference between the contents of two archives. + * #1248: Go back to treating Borg "file not found" warnings (exit code 107) as + warnings instead of errors. Otherwise, borgmatic can error on files that a user intentionally + deletes while a backup is running. You can still override this behavior with the + "borg_exit_codes" option. See the documentation for more information: + https://torsion.org/borgmatic/how-to/customize-warnings-and-errors/ + * #1248: Un-deprecate the "source_directories_must_exist" option and default it to true, to + compensate for Borg "file not found" warnings no longer being treated as errors. * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto". * #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap" action. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index f2f23c69..c20ea439 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -179,10 +179,7 @@ def make_base_create_command( # noqa: PLR0912 return a tuple of (base Borg create command flags, Borg create command positional arguments, open pattern file handle). ''' - if config.get('source_directories_must_exist', False): - logger.warning( - 'The "source_directories_must_exist" option is deprecated and will be removed from a future release; borgmatic now errors on missing files as Borg runs' - ) + if config.get('source_directories_must_exist', True): borgmatic.borg.pattern.check_all_root_patterns_exist(patterns) patterns_file = borgmatic.borg.pattern.write_patterns_file( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 15976270..c381444c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -30,12 +30,10 @@ properties: source_directories_must_exist: type: boolean description: | - Deprecated. Replaced by borgmatic treating Borg's "backup file not - found" warning as an error by default. But if - "source_directories_must_exist" is true, then source directories - (and root pattern paths) must exist before a backup begins. If they - don't, borgmatic errors. Defaults to false. - example: true + When true, source directories (and root pattern paths) must exist + before a backup begins. If they don't, borgmatic errors. Defaults to + true. + example: false repositories: type: array items: diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 5fde85c7..da186da6 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -19,7 +19,7 @@ BORG_ERROR_EXIT_CODE_START = 2 BORG_ERROR_EXIT_CODE_END = 99 # See https://borgbackup.readthedocs.io/en/stable/internals/frontends.html#message-ids -BORG_WARNING_EXIT_CODES_TREATED_AS_ERRORS = {101, 102, 104, 105, 106, 107} +BORG_WARNING_EXIT_CODES_TREATED_AS_ERRORS = {101, 102, 104, 105, 106} class Exit_status(enum.Enum): diff --git a/docs/how-to/customize-warnings-and-errors.md b/docs/how-to/customize-warnings-and-errors.md index 72edb3c1..a2473cdd 100644 --- a/docs/how-to/customize-warnings-and-errors.md +++ b/docs/how-to/customize-warnings-and-errors.md @@ -12,10 +12,11 @@ decide how to respond. By default, Borg errors (and some warnings) result in a borgmatic error, while Borg successes don't. New in borgmatic version 2.1.0 -borgmatic elevates most Borg warnings to errors by default. For instance, if a -source directory is missing during backup, Borg indicates that with a warning -exit code (`107`). And starting in borgmatic 2.1.0, that exit code is considered -an error, so you'll actually find out about missing files. +borgmatic elevates several Borg warnings to errors by default. For instance, if +borgmatic doesn't have permission to read a configured source directory during +backup, Borg indicates that with a warning exit code (`105`). And starting in +borgmatic 2.1.0, that exit code is considered an error, so you'll actually find +out about files that borgmatic can't read. With Borg version 1.4+ If the default behavior isn't sufficient for your needs, you can customize how @@ -23,7 +24,7 @@ borgmatic interprets [Borg's exit codes](https://borgbackup.readthedocs.io/en/stable/internals/frontends.html#message-ids). For instance, this borgmatic configuration elevates a Borg warning about source files -changes during backup (exit code `100`)—and only those warnings—to +changing during backup (exit code `100`)—and only those warnings—to errors: ```yaml @@ -32,14 +33,15 @@ borg_exit_codes: treat_as: error ``` -The following configuration does that *and* treats Borg's backup file not found -(exit code `107`) as a warning: +The following configuration does that *and* squashes errors about Borg +encountering file permissions issues during backup (exit code `105`) to +warnings. ```yaml borg_exit_codes: - code: 100 treat_as: error - - code: 107 + - code: 105 treat_as: warning ``` @@ -54,8 +56,11 @@ is not found: terminating with warning status, rc 107 ``` -So if you want to configure borgmatic to treat this as an warning instead of an -error, the exit status to use is `107`. +So if you want to configure borgmatic's interpretation of this warning, the exit +status to use is `107`. Note however that in the particular case of missing +files, there's a separate [`source_directories_must_exist` +option](https://torsion.org/borgmatic/reference/configuration/#source_directories_must_exist-option) +that can catch such problems before Borg even runs. With Borg version 1.2 and earlier Older versions of Borg didn't support granular exit codes, but still diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 969b707e..2e21e648 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -357,7 +357,8 @@ DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' REPO_ARCHIVE = (f'repo::{DEFAULT_ARCHIVE_NAME}',) -def test_make_base_create_produces_borg_command(): +def test_make_base_create_command_checks_root_patterns_exist_and_produces_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist').once() flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -386,7 +387,39 @@ def test_make_base_create_produces_borg_command(): assert not pattern_file +def test_make_base_create_command_without_check_all_root_patterns_exist_skips_check_and_produces_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist').never() + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',), + ) + flexmock(module).should_receive('validate_planned_backup_paths').and_return(()) + + (create_flags, create_positional_arguments, pattern_file) = module.make_base_create_command( + dry_run=False, + repository_path='repo', + config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'source_directories_must_exist': False, + }, + patterns=[Pattern('foo'), Pattern('bar')], + local_borg_version='1.2.3', + global_arguments=flexmock(), + borgmatic_runtime_directory='/run/borgmatic', + ) + + assert create_flags == ('borg', 'create', '--log-json') + assert create_positional_arguments == REPO_ARCHIVE + assert not pattern_file + + def test_make_base_create_command_includes_patterns_file_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) mock_pattern_file = flexmock(name='/tmp/patterns') flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return( @@ -424,6 +457,7 @@ def test_make_base_create_command_includes_patterns_file_in_borg_command(): def test_make_base_create_command_with_store_config_false_omits_config_files(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -494,6 +528,7 @@ def test_make_base_create_command_includes_configuration_option_as_command_flag( feature_available, option_flags, ): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -527,6 +562,7 @@ def test_make_base_create_command_includes_configuration_option_as_command_flag( def test_make_base_create_command_with_progress_omits_log_json_from_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -561,6 +597,7 @@ def test_make_base_create_command_with_progress_omits_log_json_from_borg_command def test_make_base_create_command_with_log_json_and_progress_includes_log_json_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -596,6 +633,7 @@ def test_make_base_create_command_with_log_json_and_progress_includes_log_json_i def test_make_base_create_command_includes_dry_run_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -629,6 +667,7 @@ def test_make_base_create_command_includes_dry_run_in_borg_command(): def test_make_base_create_command_includes_comment_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -663,6 +702,7 @@ def test_make_base_create_command_includes_comment_in_borg_command(): def test_make_base_create_command_includes_local_path_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -696,6 +736,7 @@ def test_make_base_create_command_includes_local_path_in_borg_command(): def test_make_base_create_command_includes_remote_path_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -729,6 +770,7 @@ def test_make_base_create_command_includes_remote_path_in_borg_command(): def test_make_base_create_command_includes_list_flags_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -764,6 +806,7 @@ def test_make_base_create_command_includes_list_flags_in_borg_command(): def test_make_base_create_command_with_stream_processes_ignores_read_special_false_and_excludes_special_files(): patterns = [Pattern('foo'), Pattern('bar')] patterns_file = flexmock(name='patterns') + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args( patterns, @@ -823,6 +866,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal def test_make_base_create_command_without_patterns_and_with_stream_processes_ignores_read_special_false_and_excludes_special_files(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args( [], @@ -882,6 +926,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign def test_make_base_create_command_with_stream_processes_and_read_special_true_skips_special_files_excludes(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -917,6 +962,7 @@ def test_make_base_create_command_with_stream_processes_and_read_special_true_sk def test_make_base_create_command_includes_archive_name_format_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -950,6 +996,7 @@ def test_make_base_create_command_includes_archive_name_format_in_borg_command() def test_make_base_create_command_includes_default_archive_name_format_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -983,6 +1030,7 @@ def test_make_base_create_command_includes_default_archive_name_format_in_borg_c def test_make_base_create_command_includes_archive_name_format_with_placeholders_in_borg_command(): repository_archive_pattern = 'repo::Documents_{hostname}-{now}' + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.flags).should_receive('get_default_archive_name_format').and_return( @@ -1016,6 +1064,7 @@ def test_make_base_create_command_includes_archive_name_format_with_placeholders def test_make_base_create_command_includes_repository_and_archive_name_format_with_placeholders_in_borg_command(): repository_archive_pattern = '{fqdn}::Documents_{hostname}-{now}' + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.flags).should_receive('get_default_archive_name_format').and_return( @@ -1048,6 +1097,7 @@ def test_make_base_create_command_includes_repository_and_archive_name_format_wi def test_make_base_create_command_includes_archive_suffix_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -1077,6 +1127,7 @@ def test_make_base_create_command_includes_archive_suffix_in_borg_command(): def test_make_base_create_command_includes_extra_borg_options_in_borg_command(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -1117,6 +1168,7 @@ def test_make_base_create_command_includes_extra_borg_options_in_borg_command(): def test_make_base_create_command_with_unsafe_skip_path_validation_before_create_skips_validation(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') @@ -1146,6 +1198,7 @@ def test_make_base_create_command_with_unsafe_skip_path_validation_before_create def test_make_base_create_command_without_unsafe_skip_path_validation_before_create_calls_validation(): + flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO') diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 4807002a..4ea4d1ee 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -62,8 +62,8 @@ def test_command_is_borg_matches_local_path_to_command(command, borg_local_path, (True, 105, [{'code': 105, 'treat_as': 'warning'}], module.Exit_status.WARNING), (True, 106, [], module.Exit_status.ERROR), (True, 106, [{'code': 106, 'treat_as': 'warning'}], module.Exit_status.WARNING), - (True, 107, [], module.Exit_status.ERROR), - (True, 107, [{'code': 107, 'treat_as': 'warning'}], module.Exit_status.WARNING), + (True, 107, [], module.Exit_status.WARNING), + (True, 107, [{'code': 107, 'treat_as': 'error'}], module.Exit_status.ERROR), ), ) def test_interpret_exit_code_respects_exit_code_and_borg_local_path(