Add a "diff" action for viewing the difference between two archives (#1243).

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1253
Reviewed-by: Dan Helfman <witten@torsion.org>

borgmatic/actions/diff.py

@@ -0,0 +1,52 @@
+import logging
+
+import borgmatic.actions.pattern
+import borgmatic.borg.diff
+
+logger = logging.getLogger(__name__)
+
+
+def run_diff(
+    repository,
+    config,
+    local_borg_version,
+    diff_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "diff" action for the given repository.
+    '''
+
+    # Only process patterns if only_patterns flag is set
+    if diff_arguments.only_patterns:
+        processed_patterns = borgmatic.actions.pattern.process_patterns(
+            (*borgmatic.actions.pattern.collect_patterns(config),),
+            config,
+            borgmatic.config.paths.get_working_directory(config),
+        )
+    else:
+        processed_patterns = None
+
+    archive = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        diff_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.borg.diff.diff(
+        repository['path'],
+        archive,
+        config,
+        local_borg_version,
+        diff_arguments,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+        patterns=processed_patterns,
+    )

borgmatic/borg/diff.py

@@ -0,0 +1,96 @@
+import logging
+import shlex
+
+import borgmatic.borg.environment
+import borgmatic.borg.feature
+import borgmatic.config.paths
+import borgmatic.execute
+from borgmatic.borg import flags
+from borgmatic.borg.pattern import write_patterns_file
+
+logger = logging.getLogger(__name__)
+
+
+def diff(
+    repository,
+    archive,
+    config,
+    local_borg_version,
+    diff_arguments,
+    global_arguments,
+    local_path,
+    remote_path=None,
+    patterns=None,
+):
+    '''
+    Given a local or remote repository path, an archive name, a configuration dict, the local Borg
+    version string, an argparse.Namespace of diff arguments, an argparse.Namespace of global
+    arguments, optional local and remote Borg paths, executes the diff command with the given
+    arguments.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    lock_wait = config.get('lock_wait', None)
+    exclude_flags = flags.make_exclude_flags(config)
+    extra_borg_options = config.get('extra_borg_options', {}).get('diff', '')
+
+    if diff_arguments.only_patterns:
+        # Write patterns to a temporary file and use that file with --patterns-from.
+        patterns_file = write_patterns_file(
+            patterns,
+            borgmatic.config.paths.get_working_directory(config),
+        )
+    else:
+        patterns_file = None
+
+    if borgmatic.borg.feature.available(
+        borgmatic.borg.feature.Feature.NUMERIC_IDS, local_borg_version
+    ):
+        numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
+    else:
+        numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
+
+    diff_command = (
+        (local_path, 'diff')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + ('--log-json',)
+        + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (
+            ('--patterns-from', patterns_file.name)
+            if patterns_file and diff_arguments.only_patterns
+            else ()
+        )
+        + exclude_flags
+        + numeric_ids_flags
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + (
+            (
+                flags.make_repository_flags(repository, local_borg_version)
+                + flags.make_match_archives_flags(
+                    archive or config.get('match_archives'),
+                    config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE,
+                local_borg_version,
+            )
+            else flags.make_repository_archive_flags(repository, archive, local_borg_version)
+        )
+        + (('--same-chunker-params',) if diff_arguments.same_chunker_params else ())
+        + (('--sort-by', ','.join(diff_arguments.sort_keys)) if diff_arguments.sort_keys else ())
+        + (('--content-only',) if diff_arguments.content_only else ())
+        + (diff_arguments.second_archive,)
+    )
+
+    borgmatic.execute.execute_command(
+        full_command=diff_command,
+        output_log_level=logging.ANSWER,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/commands/arguments.py

@@ -33,6 +33,7 @@ ACTION_ALIASES = {
     'key': [],
     'borg': [],
     'recreate': [],
+    'diff': [],
 }
 
 
@@ -1993,6 +1994,47 @@ def make_parsers(schema, unparsed_arguments):  # noqa: PLR0915
         help='Show this help message and exit',
     )
 
+    diff_parser = action_parsers.add_parser(
+        'diff',
+        aliases=ACTION_ALIASES['diff'],
+        help='This command finds differences (file contents, user/group/mode) between archives',
+        description='This command finds differences (file contents, user/group/mode) between archives',
+    )
+    diff_group = diff_parser.add_argument_group('diff arguments')
+    diff_group.add_argument(
+        '--repository',
+        help='Path of repository containing archive to diff, defaults to the configured repository if there is only one, quoted globs supported',
+    )
+    diff_group.add_argument(
+        '--archive',
+        help='Archive name, hash, or series to diff',
+        required=True,
+    )
+    diff_group.add_argument(
+        '--second-archive',
+        help='Second archive name, hash, or series to diff',
+        required=True,
+    )
+    diff_group.add_argument(
+        '--same-chunker-params', action='store_true', help='Override check of chunker parameters'
+    )
+    diff_group.add_argument(
+        '--sort-by',
+        metavar='KEY',
+        dest='sort_keys',
+        action='append',
+        help='Advanced sorting: specify field(s) to sort by. Prefix with > for descending or < for ascending (default)',
+    )
+    diff_group.add_argument(
+        '--content-only',
+        action='store_true',
+        help='Only compare differences in content (exclude metadata differences)',
+    )
+    diff_group.add_argument(
+        '--only-patterns',
+        action='store_true',
+        help='Run the diff according to borgmatic configured patterns (ie do not diff entire archives)',
+    )
     borg_parser = action_parsers.add_parser(
         'borg',
         aliases=ACTION_ALIASES['borg'],

borgmatic/commands/borgmatic.py

@@ -21,6 +21,7 @@ import borgmatic.actions.config.show
 import borgmatic.actions.config.validate
 import borgmatic.actions.create
 import borgmatic.actions.delete
+import borgmatic.actions.diff
 import borgmatic.actions.export_key
 import borgmatic.actions.export_tar
 import borgmatic.actions.extract
@@ -623,6 +624,16 @@ def run_actions(  # noqa: PLR0912, PLR0915
                         local_path,
                         remote_path,
                     )
+                elif action_name == 'diff':
+                    borgmatic.actions.diff.run_diff(
+                        repository,
+                        config,
+                        local_borg_version,
+                        action_arguments,
+                        global_arguments,
+                        local_path,
+                        remote_path,
+                    )
                 elif action_name == 'borg':
                     borgmatic.actions.borg.run_borg(
                         repository,

borgmatic/config/schema.yaml

@@ -1115,6 +1115,7 @@ properties:
                 - break-lock
                 - key
                 - borg
+                - diff
         description: |
             List of one or more actions to skip running for this configuration
             file, even if specified on the command-line (explicitly or
@@ -1331,6 +1332,7 @@ properties:
                                   - break-lock
                                   - key
                                   - borg
+                                  - diff
                           description: |
                               List of actions for which the commands will be
                               run. Defaults to running for all actions.
@@ -1396,6 +1398,7 @@ properties:
                                   - break-lock
                                   - key
                                   - borg
+                                  - diff
                           description: |
                               Only trigger the hook when borgmatic is run with
                               particular actions listed here. Defaults to

docs/Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && borgmatic config generate --destination /etc/borgmatic --split && chmod +r /etc/borgmatic/*.yaml
 RUN mkdir /command-line \
     && borgmatic --help > /command-line/global.txt \
-    && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" "config show" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \
+    && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" "config show" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate diff borg; do \
            borgmatic $action --help > /command-line/${action/ /-}.txt; done
 RUN /app/docs/fetch-contributors >> /contributors.html
 

docs/how-to/inspect-your-backups.md

@@ -116,3 +116,22 @@ By default, borgmatic only logs to the console. But to enable simultaneous
 syslog or file logging, see the [logging
 documentation](https://torsion.org/borgmatic/reference/command-line/logging/)
 for details.
+
+## Finding differences between two archives
+
+<span class="minilink minilink-addedin">New in borgmatic version
+2.1.3</span>You can compare differences between two archives. For example:
+
+```bash
+borgmatic diff --archive latest --second-archive host-2023-01-02T04:06:07.080910
+```
+
+This shows the differences (file contents, user/group/mode) between the latest
+archive and the second one supplied.
+
+Note that, by default, `borgmatic diff` compares everything in the archives; that
+is, patterns are _not_ taken into consideration. If you require this, supply the
+`--only-patterns` flag.
+
+See the [`borgbackup`](https://borgbackup.readthedocs.io/en/stable/usage/diff.html)
+documentation for information on output format, what is compared, and more.

docs/reference/command-line/actions/diff.md

@@ -0,0 +1,16 @@
+---
+title: diff
+eleventyNavigation:
+  key: diff
+  parent: 🎬 Actions
+---
+
+{% include snippet/command-line/sample.md %}
+
+```
+{% include borgmatic/command-line/diff.txt %}
+```
+
+## Related documentation
+
+* [Finding differences](https://torsion.org/borgmatic/how-to/inspect-your-backups/#finding-differences-between-two-archives)

tests/unit/actions/test_diff.py

@@ -0,0 +1,63 @@
+from flexmock import flexmock
+
+from borgmatic.actions import diff as module
+
+
+def test_run_diff_calls_borg_diff():
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock(),
+    )
+    flexmock(module.borgmatic.borg.diff).should_receive('diff').once()
+
+    module.borgmatic.actions.diff.run_diff(
+        repository={'path': 'repo'},
+        config={},
+        local_borg_version=None,
+        diff_arguments=flexmock(
+            archive='archive',
+            same_chunker_params=False,
+            sort_keys=[],
+            content_only=False,
+            second_archive=None,
+            only_patterns=False,
+        ),
+        global_arguments=flexmock(),
+        local_path=None,
+        remote_path=None,
+    )
+
+
+def test_run_diff_with_only_patterns():
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock(),
+    )
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').once().and_return(
+        []
+    )
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').once().and_return(
+        []
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+    flexmock(module.borgmatic.borg.diff).should_receive('diff').once()
+
+    module.borgmatic.actions.diff.run_diff(
+        repository={'path': 'repo'},
+        config={},
+        local_borg_version=None,
+        diff_arguments=flexmock(
+            archive='archive',
+            same_chunker_params=False,
+            sort_keys=[],
+            content_only=False,
+            second_archive=None,
+            only_patterns=True,
+        ),
+        global_arguments=flexmock(),
+        local_path=None,
+        remote_path=None,
+    )

tests/unit/borg/test_diff.py

@@ -0,0 +1,159 @@
+import logging
+
+from flexmock import flexmock
+
+from borgmatic.borg import diff as module
+
+
+def test_diff_calls_execute_command():
+    flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(False)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive(
+        'make_repository_archive_flags'
+    ).and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(
+        flexmock(name='test')
+    )
+    flexmock(module.borgmatic.execute).should_receive('execute_command').once()
+    flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return({})
+
+    module.borgmatic.borg.diff.diff(
+        repository='repo',
+        archive='archive',
+        config={},
+        local_borg_version=None,
+        diff_arguments=flexmock(
+            same_chunker_params=False,
+            sort_keys=[],
+            content_only=False,
+            second_archive=None,
+            only_patterns=False,
+        ),
+        global_arguments=flexmock(),
+        local_path='borg',
+        remote_path=None,
+        patterns=[],
+    )
+
+
+def test_diff_with_numeric_ids_flag():
+    flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive(
+        'make_repository_archive_flags'
+    ).and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(
+        flexmock(name='test')
+    )
+    flexmock(module.borgmatic.execute).should_receive('execute_command').once()
+    flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return({})
+
+    module.borgmatic.borg.diff.diff(
+        repository='repo',
+        archive='archive',
+        config={'numeric_ids': True},
+        local_borg_version=None,
+        diff_arguments=flexmock(
+            same_chunker_params=False,
+            sort_keys=[],
+            content_only=False,
+            second_archive=None,
+            only_patterns=False,
+        ),
+        global_arguments=flexmock(),
+        local_path='borg',
+        remote_path=None,
+        patterns=[],
+    )
+
+
+def test_diff_with_numeric_ids_flag_false():
+    flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive(
+        'make_repository_archive_flags'
+    ).and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(
+        flexmock(name='test')
+    )
+    flexmock(module.borgmatic.execute).should_receive('execute_command').once()
+    flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return({})
+
+    module.borgmatic.borg.diff.diff(
+        repository='repo',
+        archive='archive',
+        config={'numeric_ids': False},
+        local_borg_version=None,
+        diff_arguments=flexmock(
+            same_chunker_params=False,
+            sort_keys=[],
+            content_only=False,
+            second_archive=None,
+            only_patterns=False,
+        ),
+        global_arguments=flexmock(),
+        local_path='borg',
+        remote_path=None,
+        patterns=[],
+    )
+
+
+def test_diff_with_only_patterns():
+    # Mock the feature check
+    flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
+
+    flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return(
+        ('--repo', 'repo')
+    )
+    flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive(
+        'make_repository_archive_flags'
+    ).and_return(())
+
+    environment = flexmock()
+    flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
+        environment
+    )
+
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(
+        '/tmp/test_patterns'
+    )
+
+    expected_command = (
+        'borg',
+        'diff',
+        '--log-json',  # Add this flag
+        '--repo',
+        'repo',
+        None,
+    )
+
+    flexmock(module.borgmatic.execute).should_receive('execute_command').with_args(
+        full_command=expected_command,
+        output_log_level=logging.ANSWER,
+        environment=environment,
+        working_directory=None,
+        borg_local_path='borg',
+        borg_exit_codes=None,
+    ).once()
+
+    module.borgmatic.borg.diff.diff(
+        repository='repo',
+        archive='archive',
+        config={},
+        local_borg_version=None,
+        diff_arguments=flexmock(
+            same_chunker_params=False,
+            sort_keys=[],
+            content_only=False,
+            second_archive=None,
+            only_patterns=True,
+        ),
+        global_arguments=flexmock(),
+        local_path='borg',
+        remote_path=None,
+        patterns=[],
+    )

tests/unit/commands/test_borgmatic.py

@@ -2689,3 +2689,27 @@ def test_get_singular_option_value_with_no_config_returns_none():
         )
         is None
     )
+
+
+def test_run_actions_runs_diff():
+    flexmock(module).should_receive('add_custom_log_levels')
+    flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock(),
+    )
+    flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
+    flexmock(borgmatic.actions.diff).should_receive('run_diff').once()
+
+    tuple(
+        module.run_actions(
+            arguments={'global': flexmock(dry_run=False), 'diff': flexmock()},
+            config_filename=flexmock(),
+            config={'repositories': []},
+            config_paths=[],
+            local_path=flexmock(),
+            remote_path=flexmock(),
+            local_borg_version=flexmock(),
+            repository={'path': 'repo'},
+        ),
+    )