Add "unsafe_create_without_precheck" option to skip pre-backup validation (#1221).

This option allows users to skip the dry-run validation that checks for
special files and runtime directory exclusions. This can significantly
improve performance for large filesystems with millions of files, where
the validation can take hours.

The option name includes "unsafe" to communicate risk, and a warning is
logged on each backup showing the runtime directory path and advising
users to verify their exclude patterns manually.

borgmatic/borg/create.py

@@ -138,7 +138,7 @@ def validate_planned_backup_paths(
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
 
 
-def make_base_create_command(
+def make_base_create_command(  # noqa: PLR0912
     dry_run,
     repository_path,
     config,
@@ -249,16 +249,25 @@ def make_base_create_command(
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    logger.debug('Checking file paths Borg plans to include')
-    planned_backup_paths = validate_planned_backup_paths(
-        dry_run,
-        create_flags + create_positional_arguments,
-        config,
-        patterns,
-        local_path,
-        working_directory,
-        borgmatic_runtime_directory=borgmatic_runtime_directory,
-    )
+    if config.get('unsafe_create_without_precheck'):
+        logger.warning(
+            'Skipping pre-backup safety checks due to "unsafe_create_without_precheck" option. '
+            'If using database hooks: (1) Borg may hang on special files, and (2) database dumps '
+            'may be silently excluded if your excludes match the runtime directory '
+            f'({borgmatic_runtime_directory}). Verify your exclude patterns manually.'
+        )
+        planned_backup_paths = ()
+    else:
+        logger.debug('Checking file paths Borg plans to include')
+        planned_backup_paths = validate_planned_backup_paths(
+            dry_run,
+            create_flags + create_positional_arguments,
+            config,
+            patterns,
+            local_path,
+            working_directory,
+            borgmatic_runtime_directory=borgmatic_runtime_directory,
+        )
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.

borgmatic/config/schema.yaml

@@ -144,6 +144,22 @@ properties:
             false. But when a database hook is used, the setting here is ignored
             and read_special is considered true.
         example: true
+    unsafe_create_without_precheck:
+        type: boolean
+        description: |
+            Skip pre-backup safety checks: (1) runtime directory exclusion
+            detection, and (2) special file exclusion for database streaming.
+            Can significantly improve performance for large filesystems.
+
+            UNSAFE because if using database hooks: Borg may hang on special
+            files (named pipes, devices), and database dumps may be silently
+            excluded if your exclude_patterns match the runtime directory
+            (see "user_runtime_directory" option, defaults to $XDG_RUNTIME_DIR
+            or $TMPDIR/borgmatic-*).
+
+            Only enable if not using database hooks, or after manually verifying
+            your excludes don't affect the runtime directory. Defaults to false.
+        example: true
     flags:
         type: boolean
         description: |

tests/unit/borg/test_create.py

@@ -1019,6 +1019,64 @@ def test_make_base_create_command_includes_extra_borg_options_in_borg_command():
     assert not pattern_file
 
 
+def test_make_base_create_command_with_unsafe_create_without_precheck_skips_validation():
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
+        '{hostname}',
+    )
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{module.flags.get_default_archive_name_format()}',),
+    )
+    flexmock(module).should_receive('validate_planned_backup_paths').never()
+    flexmock(module.logger).should_receive('warning').once()
+
+    module.make_base_create_command(
+        dry_run=False,
+        repository_path='repo',
+        config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'unsafe_create_without_precheck': True,
+        },
+        patterns=[Pattern('foo'), Pattern('bar')],
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
+def test_make_base_create_command_without_unsafe_create_without_precheck_calls_validation():
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
+        '{hostname}',
+    )
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{module.flags.get_default_archive_name_format()}',),
+    )
+    flexmock(module).should_receive('validate_planned_backup_paths').once().and_return(())
+
+    module.make_base_create_command(
+        dry_run=False,
+        repository_path='repo',
+        config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+        },
+        patterns=[Pattern('foo'), Pattern('bar')],
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
 def test_make_base_create_command_with_non_existent_directory_and_source_directories_must_exist_raises():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module.borgmatic.borg.pattern).should_receive(