Add "unsafe_skip_path_validation_before_create" option to skip pre-backup validation (#1221).

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1222

borgmatic/borg/create.py

@@ -138,7 +138,7 @@ def validate_planned_backup_paths(
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
 
 
-def make_base_create_command(
+def make_base_create_command(  # noqa: PLR0912
     dry_run,
     repository_path,
     config,
@@ -249,16 +249,22 @@ def make_base_create_command(
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    logger.debug('Checking file paths Borg plans to include')
-    planned_backup_paths = validate_planned_backup_paths(
-        dry_run,
-        create_flags + create_positional_arguments,
-        config,
-        patterns,
-        local_path,
-        working_directory,
-        borgmatic_runtime_directory=borgmatic_runtime_directory,
-    )
+    if config.get('unsafe_skip_path_validation_before_create'):
+        logger.warning(
+            'Skipping pre-backup path validation due to "unsafe_skip_path_validation_before_create" option.'
+        )
+        planned_backup_paths = ()
+    else:
+        logger.debug('Checking file paths Borg plans to include')
+        planned_backup_paths = validate_planned_backup_paths(
+            dry_run,
+            create_flags + create_positional_arguments,
+            config,
+            patterns,
+            local_path,
+            working_directory,
+            borgmatic_runtime_directory=borgmatic_runtime_directory,
+        )
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.

borgmatic/config/schema.yaml

@@ -144,6 +144,22 @@ properties:
             false. But when a database hook is used, the setting here is ignored
             and read_special is considered true.
         example: true
+    unsafe_skip_path_validation_before_create:
+        type: boolean
+        description: |
+            Skip pre-backup safety checks: (1) runtime directory exclusion
+            detection, and (2) special file exclusion for database streaming.
+            Can significantly improve performance for large filesystems.
+
+            UNSAFE because if using data source hooks: Borg may hang on special
+            files (named pipes, devices), and data from borgmatic data source
+            hooks may be silently excluded if your exclude_patterns match the
+            runtime directory (see "user_runtime_directory" option).
+
+            Only enable if not using data source hooks, or after manually
+            verifying your excludes don't affect the runtime directory. Defaults
+            to false.
+        example: true
     flags:
         type: boolean
         description: |

tests/unit/borg/test_create.py

@@ -1076,6 +1076,64 @@ def test_make_base_create_command_includes_extra_borg_options_in_borg_command():
     assert not pattern_file
 
 
+def test_make_base_create_command_with_unsafe_skip_path_validation_before_create_skips_validation():
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
+        '{hostname}',
+    )
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{module.flags.get_default_archive_name_format()}',),
+    )
+    flexmock(module).should_receive('validate_planned_backup_paths').never()
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.make_base_create_command(
+        dry_run=False,
+        repository_path='repo',
+        config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'unsafe_skip_path_validation_before_create': True,
+        },
+        patterns=[Pattern('foo'), Pattern('bar')],
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
+def test_make_base_create_command_without_unsafe_skip_path_validation_before_create_calls_validation():
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
+        '{hostname}',
+    )
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{module.flags.get_default_archive_name_format()}',),
+    )
+    flexmock(module).should_receive('validate_planned_backup_paths').once().and_return(())
+
+    module.make_base_create_command(
+        dry_run=False,
+        repository_path='repo',
+        config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+        },
+        patterns=[Pattern('foo'), Pattern('bar')],
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
 def test_make_base_create_command_with_non_existent_directory_and_source_directories_must_exist_raises():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive(