Deduplicate overlapping source directories and patterns so they don't throw off "spot" check file counts and cause spurious check failures.

NEWS

@@ -7,6 +7,8 @@
  * #1260: Fix for SSH warnings from Borg showing up as JSON logs even without the "--log-json" flag.
  * #1252: Work around Borg returning a warning exit code when a repository/archive check fails. Now,
    borgmatic interprets such failures as errors.
+ * Deduplicate overlapping source directories and patterns so they don't throw off "spot" check file
+   counts and cause spurious check failures.
 
 2.1.1
  * #1241: For the "recreate" action, actually pass the "--dry-run" flag through to Borg instead of

borgmatic/actions/check.py

@@ -418,7 +418,13 @@ def collect_spot_check_source_paths(
     )
 
     return tuple(
-        path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
+        # Use dict.fromkeys() to deduplicate file paths, which are present in Borg's dry run output
+        # when there are overlapping source patterns. For instance, if both "/foo" and
+        # "/foo/file.txt" are in configured patterns, then "/foo/file.txt" will show up in Borg's
+        # dry run output twice.
+        dict.fromkeys(
+            path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
+        )
     )
 
 

tests/unit/actions/test_check.py

@@ -986,6 +986,65 @@ def test_collect_spot_check_source_paths_uses_working_directory():
     ) == ('foo', 'bar')
 
 
+def test_collect_spot_check_source_paths_deduplicates_borg_output_paths():
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
+        {'hook1': False, 'hook2': True},
+    )
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock(),
+    )
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
+        (Pattern('collected'),),
+    )
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').with_args(
+        (
+            Pattern('collected', source=module.borgmatic.borg.pattern.Pattern_source.HOOK),
+            Pattern('extra.yaml', source=module.borgmatic.borg.pattern.Pattern_source.INTERNAL),
+        ),
+        config=object,
+        working_directory=None,
+    ).and_return(
+        [Pattern('foo'), Pattern('bar')],
+    )
+    flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
+        dry_run=True,
+        repository_path='repo',
+        config=object,
+        patterns=[Pattern('foo'), Pattern('bar')],
+        local_borg_version=object,
+        global_arguments=object,
+        borgmatic_runtime_directory='/run/borgmatic',
+        local_path=object,
+        remote_path=object,
+        stream_processes=True,
+    ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
+    flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
+        flexmock(),
+    )
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output',
+    ).and_yield(
+        'warning: stuff',
+        '- /etc/path',
+        '+ /etc/other',
+        '? /nope',
+        '- /etc/path',
+    )
+    flexmock(module.os.path).should_receive('isfile').and_return(True)
+
+    assert module.collect_spot_check_source_paths(
+        repository={'path': 'repo'},
+        config={'working_directory': '/'},
+        local_borg_version=flexmock(),
+        global_arguments=flexmock(),
+        local_path=flexmock(),
+        remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+        bootstrap_config_paths=('extra.yaml',),
+    ) == ('/etc/path', '/etc/other')
+
+
 def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
     flexmock(module.random).should_receive('SystemRandom').and_return(
         flexmock(sample=lambda population, count: population[:count]),