Treat Borg "file not found" warnings (exit code 107) as warnings again instead of errors. Also un-deprecate the "source_directories_must_exist" option and default it to true (#1248).

NEWS

@@ -8,6 +8,13 @@
  * #1228: Adjust the "spot" check so error output includes more information about what failed.
  * #1236: Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them.
  * #1243: Add a "diff" action for viewing the difference between the contents of two archives.
+ * #1248: Go back to treating Borg "file not found" warnings (exit code 107) as
+   warnings instead of errors. Otherwise, borgmatic can error on files that a user intentionally
+   deletes while a backup is running. You can still override this behavior with the
+   "borg_exit_codes" option. See the documentation for more information:
+   https://torsion.org/borgmatic/how-to/customize-warnings-and-errors/
+ * #1248: Un-deprecate the "source_directories_must_exist" option and default it to true, to
+   compensate for Borg "file not found" warnings no longer being treated as errors.
  * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto".
  * #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap"
    action.

borgmatic/borg/create.py

@@ -179,10 +179,7 @@ def make_base_create_command(  # noqa: PLR0912
     return a tuple of (base Borg create command flags, Borg create command positional arguments,
     open pattern file handle).
     '''
-    if config.get('source_directories_must_exist', False):
-        logger.warning(
-            'The "source_directories_must_exist" option is deprecated and will be removed from a future release; borgmatic now errors on missing files as Borg runs'
-        )
+    if config.get('source_directories_must_exist', True):
         borgmatic.borg.pattern.check_all_root_patterns_exist(patterns)
 
     patterns_file = borgmatic.borg.pattern.write_patterns_file(

borgmatic/config/schema.yaml

@@ -30,12 +30,10 @@ properties:
     source_directories_must_exist:
         type: boolean
         description: |
-            Deprecated. Replaced by borgmatic treating Borg's "backup file not
-            found" warning as an error by default. But if
-            "source_directories_must_exist" is true, then source directories
-            (and root pattern paths) must exist before a backup begins. If they
-            don't, borgmatic errors. Defaults to false.
-        example: true
+            When true, source directories (and root pattern paths) must exist
+            before a backup begins. If they don't, borgmatic errors. Defaults to
+            true.
+        example: false
     repositories:
         type: array
         items:

borgmatic/execute.py

@@ -19,7 +19,7 @@ BORG_ERROR_EXIT_CODE_START = 2
 BORG_ERROR_EXIT_CODE_END = 99
 
 # See https://borgbackup.readthedocs.io/en/stable/internals/frontends.html#message-ids
-BORG_WARNING_EXIT_CODES_TREATED_AS_ERRORS = {101, 102, 104, 105, 106, 107}
+BORG_WARNING_EXIT_CODES_TREATED_AS_ERRORS = {101, 102, 104, 105, 106}
 
 
 class Exit_status(enum.Enum):

docs/how-to/customize-warnings-and-errors.md

@@ -12,10 +12,11 @@ decide how to respond. By default, Borg errors (and some warnings) result
 in a borgmatic error, while Borg successes don't.
 
 <span class="minilink minilink-addedin">New in borgmatic version 2.1.0</span>
-borgmatic elevates most Borg warnings to errors by default. For instance, if a
-source directory is missing during backup, Borg indicates that with a warning
-exit code (`107`). And starting in borgmatic 2.1.0, that exit code is considered
-an error, so you'll actually find out about missing files.
+borgmatic elevates several Borg warnings to errors by default. For instance, if
+borgmatic doesn't have permission to read a configured source directory during
+backup, Borg indicates that with a warning exit code (`105`). And starting in
+borgmatic 2.1.0, that exit code is considered an error, so you'll actually find
+out about files that borgmatic can't read.
 
 <span class="minilink minilink-addedin">With Borg version 1.4+</span> If the
 default behavior isn't sufficient for your needs, you can customize how
@@ -23,7 +24,7 @@ borgmatic interprets [Borg's exit
 codes](https://borgbackup.readthedocs.io/en/stable/internals/frontends.html#message-ids).
 
 For instance, this borgmatic configuration elevates a Borg warning about source files
-changes during backup (exit code `100`)—and only those warnings—to
+changing during backup (exit code `100`)—and only those warnings—to
 errors:
 
 ```yaml
@@ -32,14 +33,15 @@ borg_exit_codes:
       treat_as: error
 ```
 
-The following configuration does that *and* treats Borg's backup file not found
-(exit code `107`) as a warning:
+The following configuration does that *and* squashes errors about Borg
+encountering file permissions issues during backup (exit code `105`) to
+warnings.
 
 ```yaml
 borg_exit_codes:
     - code: 100
       treat_as: error
-    - code: 107
+    - code: 105
       treat_as: warning
 ```
 
@@ -54,8 +56,11 @@ is not found:
 terminating with warning status, rc 107
 ```
 
-So if you want to configure borgmatic to treat this as an warning instead of an
-error, the exit status to use is `107`.
+So if you want to configure borgmatic's interpretation of this warning, the exit
+status to use is `107`. Note however that in the particular case of missing
+files, there's a separate [`source_directories_must_exist`
+option](https://torsion.org/borgmatic/reference/configuration/#source_directories_must_exist-option)
+that can catch such problems before Borg even runs.
 
 <span class="minilink minilink-addedin">With Borg version 1.2 and earlier</span>
 Older versions of Borg didn't support granular exit codes, but still

tests/unit/borg/test_create.py

@@ -357,7 +357,8 @@ DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
 REPO_ARCHIVE = (f'repo::{DEFAULT_ARCHIVE_NAME}',)
 
 
-def test_make_base_create_produces_borg_command():
+def test_make_base_create_command_checks_root_patterns_exist_and_produces_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist').once()
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -386,7 +387,39 @@ def test_make_base_create_produces_borg_command():
     assert not pattern_file
 
 
+def test_make_base_create_command_without_check_all_root_patterns_exist_skips_check_and_produces_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist').never()
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{DEFAULT_ARCHIVE_NAME}',),
+    )
+    flexmock(module).should_receive('validate_planned_backup_paths').and_return(())
+
+    (create_flags, create_positional_arguments, pattern_file) = module.make_base_create_command(
+        dry_run=False,
+        repository_path='repo',
+        config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'source_directories_must_exist': False,
+        },
+        patterns=[Pattern('foo'), Pattern('bar')],
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+    assert create_flags == ('borg', 'create', '--log-json')
+    assert create_positional_arguments == REPO_ARCHIVE
+    assert not pattern_file
+
+
 def test_make_base_create_command_includes_patterns_file_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     mock_pattern_file = flexmock(name='/tmp/patterns')
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(
@@ -424,6 +457,7 @@ def test_make_base_create_command_includes_patterns_file_in_borg_command():
 
 
 def test_make_base_create_command_with_store_config_false_omits_config_files():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -494,6 +528,7 @@ def test_make_base_create_command_includes_configuration_option_as_command_flag(
     feature_available,
     option_flags,
 ):
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -527,6 +562,7 @@ def test_make_base_create_command_includes_configuration_option_as_command_flag(
 
 
 def test_make_base_create_command_with_progress_omits_log_json_from_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -561,6 +597,7 @@ def test_make_base_create_command_with_progress_omits_log_json_from_borg_command
 
 
 def test_make_base_create_command_with_log_json_and_progress_includes_log_json_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -596,6 +633,7 @@ def test_make_base_create_command_with_log_json_and_progress_includes_log_json_i
 
 
 def test_make_base_create_command_includes_dry_run_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -629,6 +667,7 @@ def test_make_base_create_command_includes_dry_run_in_borg_command():
 
 
 def test_make_base_create_command_includes_comment_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -663,6 +702,7 @@ def test_make_base_create_command_includes_comment_in_borg_command():
 
 
 def test_make_base_create_command_includes_local_path_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -696,6 +736,7 @@ def test_make_base_create_command_includes_local_path_in_borg_command():
 
 
 def test_make_base_create_command_includes_remote_path_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -729,6 +770,7 @@ def test_make_base_create_command_includes_remote_path_in_borg_command():
 
 
 def test_make_base_create_command_includes_list_flags_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -764,6 +806,7 @@ def test_make_base_create_command_includes_list_flags_in_borg_command():
 def test_make_base_create_command_with_stream_processes_ignores_read_special_false_and_excludes_special_files():
     patterns = [Pattern('foo'), Pattern('bar')]
     patterns_file = flexmock(name='patterns')
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
         patterns,
@@ -823,6 +866,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
 
 
 def test_make_base_create_command_without_patterns_and_with_stream_processes_ignores_read_special_false_and_excludes_special_files():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
         [],
@@ -882,6 +926,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
 
 
 def test_make_base_create_command_with_stream_processes_and_read_special_true_skips_special_files_excludes():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -917,6 +962,7 @@ def test_make_base_create_command_with_stream_processes_and_read_special_true_sk
 
 
 def test_make_base_create_command_includes_archive_name_format_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -950,6 +996,7 @@ def test_make_base_create_command_includes_archive_name_format_in_borg_command()
 
 
 def test_make_base_create_command_includes_default_archive_name_format_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -983,6 +1030,7 @@ def test_make_base_create_command_includes_default_archive_name_format_in_borg_c
 
 def test_make_base_create_command_includes_archive_name_format_with_placeholders_in_borg_command():
     repository_archive_pattern = 'repo::Documents_{hostname}-{now}'
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
@@ -1016,6 +1064,7 @@ def test_make_base_create_command_includes_archive_name_format_with_placeholders
 
 def test_make_base_create_command_includes_repository_and_archive_name_format_with_placeholders_in_borg_command():
     repository_archive_pattern = '{fqdn}::Documents_{hostname}-{now}'
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
@@ -1048,6 +1097,7 @@ def test_make_base_create_command_includes_repository_and_archive_name_format_wi
 
 
 def test_make_base_create_command_includes_archive_suffix_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -1077,6 +1127,7 @@ def test_make_base_create_command_includes_archive_suffix_in_borg_command():
 
 
 def test_make_base_create_command_includes_extra_borg_options_in_borg_command():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -1117,6 +1168,7 @@ def test_make_base_create_command_includes_extra_borg_options_in_borg_command():
 
 
 def test_make_base_create_command_with_unsafe_skip_path_validation_before_create_skips_validation():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
@@ -1146,6 +1198,7 @@ def test_make_base_create_command_with_unsafe_skip_path_validation_before_create
 
 
 def test_make_base_create_command_without_unsafe_skip_path_validation_before_create_calls_validation():
+    flexmock(module.borgmatic.borg.pattern).should_receive('check_all_root_patterns_exist')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')

tests/unit/test_execute.py

@@ -62,8 +62,8 @@ def test_command_is_borg_matches_local_path_to_command(command, borg_local_path,
         (True, 105, [{'code': 105, 'treat_as': 'warning'}], module.Exit_status.WARNING),
         (True, 106, [], module.Exit_status.ERROR),
         (True, 106, [{'code': 106, 'treat_as': 'warning'}], module.Exit_status.WARNING),
-        (True, 107, [], module.Exit_status.ERROR),
-        (True, 107, [{'code': 107, 'treat_as': 'warning'}], module.Exit_status.WARNING),
+        (True, 107, [], module.Exit_status.WARNING),
+        (True, 107, [{'code': 107, 'treat_as': 'error'}], module.Exit_status.ERROR),
     ),
 )
 def test_interpret_exit_code_respects_exit_code_and_borg_local_path(