Fix that errors and exits when the borgmatic runtime directory is partially excluded by configured excludes (#1203).

NEWS

@@ -16,6 +16,11 @@
    global excludes.
  * #1201: Document a problematic interaction between borgmatic and systemd-tmpfiles:
    https://torsion.org/borgmatic/reference/configuration/runtime-directory/#systemd-tmpfiles
+ * #1203: Fix that errors and exits when the borgmatic runtime directory is partially excluded by
+   configured excludes. Previously, borgmatic only errored when the runtime directory was completely
+   excluded.
+ * #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always
+   error and exit when the borgmatic runtime directory overlaps with the configured excludes.
  * Update the sample systemd timer with a shorter random delay when catching up on a missed run.
 
 2.0.12

borgmatic/borg/create.py

@@ -96,32 +96,29 @@ def validate_planned_backup_paths(
         if path_line and path_line.startswith(('- ', '+ '))
     )
 
-    # These are the subset of output paths contained within the borgmatic runtime directory.
-    paths_inside_runtime_directory = {
-        path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
-    }
-
-    # If the runtime directory isn't present in the source patterns, then we shouldn't expect it to
-    # be in the paths output from the Borg dry run.
-    runtime_directory_present_in_patterns = any(
+    runtime_directory_root_patterns = tuple(
         pattern
         for pattern in patterns
         if any_parent_directories(pattern.path, (borgmatic_runtime_directory,))
         if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
     )
 
-    # If no paths to backup are inside the runtime directory, it must've been excluded.
-    if (
-        not paths_inside_runtime_directory
-        and runtime_directory_present_in_patterns
-        and not dry_run
-        and os.path.exists(borgmatic_runtime_directory)
-    ):
-        raise ValueError(
-            f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.',
-        )
+    if not dry_run and os.path.exists(borgmatic_runtime_directory):
+        # If there are any root patterns in the runtime directory that are missing from the paths
+        # Borg is planning to backup, then they must've gotten excluded, e.g. by user-configured
+        # excludes. Error accordingly.
+        for pattern in runtime_directory_root_patterns:
+            if not any(any_parent_directories(path, (pattern.path,)) for path in paths):
+                raise ValueError(
+                    f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.',
+                )
 
-    return tuple(path for path in paths if path not in paths_inside_runtime_directory)
+    # Return the subset of output paths *not* contained within the borgmatic runtime directory. The
+    # intent is that any downstream checks using these paths should skip runtime paths that
+    # borgmatic uses for its own bookkeeping, instead focusing on user-configured paths.
+    return tuple(
+        path for path in paths if not any_parent_directories(path, (borgmatic_runtime_directory,))
+    )
 
 
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000

borgmatic/config/schema.yaml

@@ -177,7 +177,9 @@ properties:
             as "source_directories"; they tell Borg which paths to backup
             (modulo any excludes). Globs are expanded. (Tildes are not.) See
             the output of "borg help patterns" for more details. Quote any value
-            if it contains leading punctuation, so it parses correctly.
+            if it contains leading punctuation, so it parses correctly. Also use
+            leading slashes in absolute paths, or data source hooks may be
+            unable to rewrite patterns as needed.
         example:
             - 'R /'
             - '- /home/*/.cache'

tests/unit/borg/test_create.py

@@ -101,7 +101,7 @@ def test_validate_planned_backup_paths_skips_borgmatic_runtime_directory():
     )
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module).should_receive('any_parent_directories').replace_with(
-        lambda path, _: path == '/run/borgmatic/bar'
+        lambda path, candidates: any(path.startswith(parent) for parent in candidates)
     )
 
     assert module.validate_planned_backup_paths(
@@ -134,7 +134,7 @@ def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_
     )
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module).should_receive('any_parent_directories').replace_with(
-        lambda path, _: path == '/run/borgmatic/bar'
+        lambda path, candidates: any(path.startswith(parent) for parent in candidates)
     )
 
     with pytest.raises(ValueError):
@@ -155,6 +155,45 @@ def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_
         )
 
 
+def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_partially_excluded_from_paths_output_errors():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments,
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments,
+    )
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
+
+    # /run/borgmatic/bar is present, but /run/borgmatic/quux is missing.
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(
+        '+ /foo\n- /run/borgmatic/bar\n- /baz',
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module).should_receive('any_parent_directories').replace_with(
+        lambda path, candidates: any(path.startswith(parent) for parent in candidates)
+    )
+
+    with pytest.raises(ValueError):
+        module.validate_planned_backup_paths(
+            dry_run=False,
+            create_command=('borg', 'create'),
+            config={},
+            patterns=(
+                module.borgmatic.borg.pattern.Pattern('/foo'),
+                module.borgmatic.borg.pattern.Pattern(
+                    '/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
+                ),
+                module.borgmatic.borg.pattern.Pattern('/baz'),
+                module.borgmatic.borg.pattern.Pattern(
+                    '/run/borgmatic/quux', module.borgmatic.borg.pattern.Pattern_type.ROOT
+                ),
+            ),
+            local_path=None,
+            working_directory=None,
+            borgmatic_runtime_directory='/run/borgmatic',
+        )
+
+
 def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_from_patterns_does_not_raise():
     flexmock(module.flags).should_receive('omit_flag').replace_with(
         lambda arguments, flag: arguments,
@@ -168,7 +207,7 @@ def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_
     )
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module).should_receive('any_parent_directories').replace_with(
-        lambda path, _: path == '/run/borgmatic/bar'
+        lambda path, candidates: any(path.startswith(parent) for parent in candidates)
     )
 
     assert module.validate_planned_backup_paths(