Bump version for release.

Merge pull request #68 from diivi/feat/add-log-filename-to-hook-context

NEWS

@@ -1,3 +1,24 @@
+1.7.12
+ * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive"
+   flags. Also fix the "--match-archives"/"--archive" flags to correctly override the
+   "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions.
+ * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix"
+   options set.
+ * #672: Selectively shallow merge certain mappings or sequences when including configuration files.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge
+ * #672: Selectively omit list values when including configuration files. See the documentation for
+   more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge
+ * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes
+ * Add optional support for running end-to-end tests and building documentation with rootless Podman
+   instead of Docker.
+
 1.7.11
  * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives
    get used for borgmatic actions that operate on multiple archives. Override this behavior with the

borgmatic/borg/info.py

@@ -46,21 +46,18 @@ def display_archives_info(
             if info_arguments.prefix
             else (
                 flags.make_match_archives_flags(
-                    storage_config.get('match_archives'),
+                    info_arguments.match_archives
+                    or info_arguments.archive
+                    or storage_config.get('match_archives'),
                     storage_config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
         )
         + flags.make_flags_from_arguments(
-            info_arguments, excludes=('repository', 'archive', 'prefix')
+            info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
         )
         + flags.make_repository_flags(repository_path, local_borg_version)
-        + (
-            flags.make_flags('match-archives', info_arguments.archive)
-            if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-            else flags.make_flags('glob-archives', info_arguments.archive)
-        )
     )
 
     if info_arguments.json:

borgmatic/borg/prune.py

@@ -26,22 +26,24 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
     config = retention_config.copy()
     prefix = config.pop('prefix', None)
 
-    if prefix:
-        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-            config['match_archives'] = f'sh:{prefix}*'
-        else:
-            config['glob_archives'] = f'{prefix}*'
-
     flag_pairs = (
         ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
     )
 
-    return tuple(
-        element for pair in flag_pairs for element in pair
-    ) + flags.make_match_archives_flags(
-        storage_config.get('match_archives'),
-        storage_config.get('archive_name_format'),
-        local_borg_version,
+    return tuple(element for pair in flag_pairs for element in pair) + (
+        (
+            ('--match-archives', f'sh:{prefix}*')
+            if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
+            else ('--glob-archives', f'{prefix}*')
+        )
+        if prefix
+        else (
+            flags.make_match_archives_flags(
+                storage_config.get('match_archives'),
+                storage_config.get('archive_name_format'),
+                local_borg_version,
+            )
+        )
     )
 
 

borgmatic/borg/rlist.py

@@ -52,7 +52,7 @@ def resolve_archive_name(
     return latest_archive
 
 
-MAKE_FLAGS_EXCLUDES = ('repository', 'prefix')
+MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
 
 
 def make_rlist_command(
@@ -96,7 +96,7 @@ def make_rlist_command(
             if rlist_arguments.prefix
             else (
                 flags.make_match_archives_flags(
-                    storage_config.get('match_archives'),
+                    rlist_arguments.match_archives or storage_config.get('match_archives'),
                     storage_config.get('archive_name_format'),
                     local_borg_version,
                 )

borgmatic/borg/transfer.py

@@ -28,12 +28,6 @@ def transfer_archives(
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('lock-wait', storage_config.get('lock_wait', None))
-        + (('--progress',) if transfer_arguments.progress else ())
-        + (
-            flags.make_flags(
-                'match-archives', transfer_arguments.match_archives or transfer_arguments.archive
-            )
-        )
         + (
             flags.make_flags_from_arguments(
                 transfer_arguments,
@@ -41,7 +35,9 @@ def transfer_archives(
             )
             or (
                 flags.make_match_archives_flags(
-                    storage_config.get('match_archives'),
+                    transfer_arguments.match_archives
+                    or transfer_arguments.archive
+                    or storage_config.get('match_archives'),
                     storage_config.get('archive_name_format'),
                     local_borg_version,
                 )

borgmatic/commands/arguments.py

@@ -652,7 +652,7 @@ def make_parsers():
         '--json', default=False, action='store_true', help='Output results as JSON'
     )
     rlist_group.add_argument(
-        '-P', '--prefix', help='Only list archive names starting with this prefix'
+        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
     )
     rlist_group.add_argument(
         '-a',
@@ -707,7 +707,7 @@ def make_parsers():
         '--json', default=False, action='store_true', help='Output results as JSON'
     )
     list_group.add_argument(
-        '-P', '--prefix', help='Only list archive names starting with this prefix'
+        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
     )
     list_group.add_argument(
         '-a',
@@ -779,7 +779,9 @@ def make_parsers():
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
     info_group.add_argument(
-        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+        '-P',
+        '--prefix',
+        help='Deprecated. Only show info for archive names starting with this prefix',
     )
     info_group.add_argument(
         '-a',
@@ -877,7 +879,17 @@ def parse_arguments(*unparsed_arguments):
         and arguments['transfer'].match_archives
     ):
         raise ValueError(
-            'With the transfer action, only one of --archive and --glob-archives flags can be used.'
+            'With the transfer action, only one of --archive and --match-archives flags can be used.'
+        )
+
+    if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives):
+        raise ValueError(
+            'With the list action, only one of --prefix or --match-archives flags can be used.'
+        )
+
+    if 'rlist' in arguments and (arguments['rlist'].prefix and arguments['rlist'].match_archives):
+        raise ValueError(
+            'With the rlist action, only one of --prefix or --match-archives flags can be used.'
         )
 
     if 'info' in arguments and (

borgmatic/commands/borgmatic.py

@@ -267,6 +267,7 @@ def run_actions(
         'repository': repository_path,
         # Deprecated: For backwards compatibility with borgmatic < 1.6.0.
         'repositories': ','.join([repo['path'] for repo in location['repositories']]),
+        'log_file': global_arguments.log_file if global_arguments.log_file else '',
     }
 
     command.execute_hook(

borgmatic/commands/validate_config.py

@@ -2,6 +2,7 @@ import logging
 import sys
 from argparse import ArgumentParser
 
+import borgmatic.config.generate
 from borgmatic.config import collect, validate
 
 logger = logging.getLogger(__name__)
@@ -23,16 +24,22 @@ def parse_arguments(*arguments):
         default=config_paths,
         help=f'Configuration filenames or directories, defaults to: {config_paths}',
     )
+    parser.add_argument(
+        '-s',
+        '--show',
+        action='store_true',
+        help='Show the validated configuration after all include merging has occurred',
+    )
 
     return parser.parse_args(arguments)
 
 
 def main():  # pragma: no cover
-    args = parse_arguments(*sys.argv[1:])
+    arguments = parse_arguments(*sys.argv[1:])
 
     logging.basicConfig(level=logging.INFO, format='%(message)s')
 
-    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths))
     if len(config_filenames) == 0:
         logger.critical('No files to validate found')
         sys.exit(1)
@@ -40,13 +47,22 @@ def main():  # pragma: no cover
     found_issues = False
     for config_filename in config_filenames:
         try:
-            validate.parse_configuration(config_filename, validate.schema_filename())
+            config, parse_logs = validate.parse_configuration(
+                config_filename, validate.schema_filename()
+            )
         except (ValueError, OSError, validate.Validation_error) as error:
             logging.critical(f'{config_filename}: Error parsing configuration file')
             logging.critical(error)
             found_issues = True
+        else:
+            for log in parse_logs:
+                logger.handle(log)
+
+            if arguments.show:
+                print('---')
+                print(borgmatic.config.generate.render_configuration(config))
 
     if found_issues:
         sys.exit(1)
-    else:
-        logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")
+
+    logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")

borgmatic/config/load.py

@@ -38,6 +38,37 @@ def include_configuration(loader, filename_node, include_directory):
     return load_configuration(include_filename)
 
 
+def raise_retain_node_error(loader, node):
+    '''
+    Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!retain" usage.
+
+    Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was
+    used in a configuration file without a merge. In configuration files with a merge, mapping and
+    sequence nodes with "!retain" tags are handled by deep_merge_nodes() below.
+
+    Also raise ValueError if a scalar node is given, as "!retain" is not supported on scalar nodes.
+    '''
+    if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)):
+        raise ValueError(
+            'The !retain tag may only be used within a configuration file containing a merged !include tag.'
+        )
+
+    raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
+
+
+def raise_omit_node_error(loader, node):
+    '''
+    Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!omit" usage.
+
+    Raise ValueError unconditionally, as an "!omit" node here indicates it was used in a
+    configuration file without a merge. In configuration files with a merge, nodes with "!omit"
+    tags are handled by deep_merge_nodes() below.
+    '''
+    raise ValueError(
+        'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.'
+    )
+
+
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
@@ -50,6 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
             '!include',
             functools.partial(include_configuration, include_directory=include_directory),
         )
+        self.add_constructor('!retain', raise_retain_node_error)
+        self.add_constructor('!omit', raise_omit_node_error)
 
     def flatten_mapping(self, node):
         '''
@@ -115,6 +148,16 @@ def load_configuration(filename):
         return config
 
 
+def filter_omitted_nodes(nodes):
+    '''
+    Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a
+    value matching such nodes.
+    '''
+    omitted_values = tuple(node.value for node in nodes if node.tag == '!omit')
+
+    return [node for node in nodes if node.value not in omitted_values]
+
+
 DELETED_NODE = object()
 
 
@@ -176,6 +219,8 @@ def deep_merge_nodes(nodes):
             ),
         ]
 
+    If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged.
+
     The purpose of deep merging like this is to support, for instance, merging one borgmatic
     configuration file into another for reuse, such that a configuration section ("retention",
     etc.) does not completely replace the corresponding section in a merged file.
@@ -198,32 +243,42 @@ def deep_merge_nodes(nodes):
 
                 # If we're dealing with MappingNodes, recurse and merge its values as well.
                 if isinstance(b_value, ruamel.yaml.nodes.MappingNode):
-                    replaced_nodes[(b_key, b_value)] = (
-                        b_key,
-                        ruamel.yaml.nodes.MappingNode(
-                            tag=b_value.tag,
-                            value=deep_merge_nodes(a_value.value + b_value.value),
-                            start_mark=b_value.start_mark,
-                            end_mark=b_value.end_mark,
-                            flow_style=b_value.flow_style,
-                            comment=b_value.comment,
-                            anchor=b_value.anchor,
-                        ),
-                    )
+                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+                    # downstream schema validation doesn't break on our application-specific tag.
+                    if b_value.tag == '!retain':
+                        b_value.tag = 'tag:yaml.org,2002:map'
+                    else:
+                        replaced_nodes[(b_key, b_value)] = (
+                            b_key,
+                            ruamel.yaml.nodes.MappingNode(
+                                tag=b_value.tag,
+                                value=deep_merge_nodes(a_value.value + b_value.value),
+                                start_mark=b_value.start_mark,
+                                end_mark=b_value.end_mark,
+                                flow_style=b_value.flow_style,
+                                comment=b_value.comment,
+                                anchor=b_value.anchor,
+                            ),
+                        )
                 # If we're dealing with SequenceNodes, merge by appending one sequence to the other.
                 elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode):
-                    replaced_nodes[(b_key, b_value)] = (
-                        b_key,
-                        ruamel.yaml.nodes.SequenceNode(
-                            tag=b_value.tag,
-                            value=a_value.value + b_value.value,
-                            start_mark=b_value.start_mark,
-                            end_mark=b_value.end_mark,
-                            flow_style=b_value.flow_style,
-                            comment=b_value.comment,
-                            anchor=b_value.anchor,
-                        ),
-                    )
+                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+                    # downstream schema validation doesn't break on our application-specific tag.
+                    if b_value.tag == '!retain':
+                        b_value.tag = 'tag:yaml.org,2002:seq'
+                    else:
+                        replaced_nodes[(b_key, b_value)] = (
+                            b_key,
+                            ruamel.yaml.nodes.SequenceNode(
+                                tag=b_value.tag,
+                                value=filter_omitted_nodes(a_value.value + b_value.value),
+                                start_mark=b_value.start_mark,
+                                end_mark=b_value.end_mark,
+                                flow_style=b_value.flow_style,
+                                comment=b_value.comment,
+                                anchor=b_value.anchor,
+                            ),
+                        )
 
     return [
         replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE

docs/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.17.1 as borgmatic
+FROM docker.io/alpine:3.17.1 as borgmatic
 
 COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
@@ -8,7 +8,7 @@ RUN borgmatic --help > /command-line.txt \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:19.5.0-alpine as html
+FROM docker.io/node:19.5.0-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -28,7 +28,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.22.1-alpine
+FROM docker.io/nginx:1.22.1-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -66,6 +66,9 @@ variables you can use here:
 
  * `configuration_filename`: borgmatic configuration filename in which the
    hook was defined
+ * `log_file`
+   <span class="minilink minilink-addedin">New in version 1.7.12</span>:
+   path of the borgmatic log file, only set when the `--log-file` flag is used
  * `repository`: path of the current repository as configured in the current
    borgmatic configuration file
 

docs/how-to/develop-on-borgmatic.md

@@ -101,6 +101,30 @@ scripts/run-end-to-end-dev-tests
 Note that this scripts assumes you have permission to run Docker. If you
 don't, then you may need to run with `sudo`.
 
+
+#### Podman
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span>
+borgmatic's end-to-end tests optionally support using
+[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md)
+[Podman](https://podman.io/) instead of Docker.
+
+Setting up Podman is outside the scope of this documentation, but here are
+some key points to double-check:
+
+ * Install Podman along with `podman-docker` and your desired networking
+   support.
+ * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the
+   non-root user who will run tests.
+ * Create a non-root Podman socket for that user:
+   ```bash
+   systemctl --user enable --now podman.socket
+   ```
+
+Then you'll be able to run end-to-end tests as per normal, and the test script
+will automatically use your non-root Podman socket instead of a Docker socket.
+
+
 ## Code style
 
 Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
@@ -149,3 +173,15 @@ http://localhost:8080 to view the documentation with your changes.
 To close the documentation server, ctrl-C the script. Note that it does not
 currently auto-reload, so you'll need to stop it and re-run it for any
 additional documentation changes to take effect.
+
+
+#### Podman
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span>
+borgmatic's developer build for documentation optionally supports using
+[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md)
+[Podman](https://podman.io/) instead of Docker.
+
+Setting up Podman is outside the scope of this documentation. But once you
+install `podman-docker`, then `scripts/dev-docs` should automatically use
+Podman instead of Docker.

docs/how-to/make-per-application-backups.md

@@ -70,7 +70,7 @@ and borgmatic uses that format to name any new archive it creates. For
 instance:
 
 ```yaml
-location:
+storage:
     ...
     archive_name_format: home-directories-{now}
 ```
@@ -96,7 +96,7 @@ to filter archives when running supported actions.
 For instance, let's say that you have this in your configuration:
 
 ```yaml
-location:
+storage:
     ...
     archive_name_format: {hostname}-user-data-{now}
 ```
@@ -116,7 +116,7 @@ If this behavior isn't quite smart enough for your needs, you can use the
 filtering archives. For example:
 
 ```yaml
-location:
+storage:
     ...
     archive_name_format: {hostname}-user-data-{now}
     match_archives: sh:myhost-user-data-*        
@@ -128,13 +128,17 @@ documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-
 for more information. For Borg 2.x, see the [match archives
 documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives).
 
+Some borgmatic command-line actions also have a `--match-archives` flag that
+overrides both the auto-matching behavior and the `match_archives`
+configuration option.
+
 <span class="minilink minilink-addedin">Prior to 1.7.11</span> The way to
 limit the archives used for the `prune` action was a `prefix` option in the
 `retention` section for matching against the start of archive names. And the
 option for limiting the archives used for the `check` action was a separate
 `prefix` in the `consistency` section. Both of these options are deprecated in
-favor of the auto-matching behavior (or `match_archives`) in newer versions of
-borgmatic.
+favor of the auto-matching behavior (or `match_archives`/`--match-archives`)
+in newer versions of borgmatic.
 
 
 ## Configuration includes
@@ -268,9 +272,140 @@ Once this include gets merged in, the resulting configuration would have a
 When there's an option collision between the local file and the merged
 include, the local file's option takes precedence.
 
+
+#### List merge
+
 <span class="minilink minilink-addedin">New in version 1.6.1</span> Colliding
 list values are appended together.
 
+<span class="minilink minilink-addedin">New in version 1.7.12</span> If there
+is a list value from an include that you *don't* want in your local
+configuration file, you can omit it with an `!omit` tag. For instance:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   source_directories:
+     - !omit /home
+     - /var
+```
+
+And `common.yaml` like this:
+
+```yaml
+location:
+   source_directories:
+     - /home
+     - /etc
+```
+
+Once this include gets merged in, the resulting configuration will have a
+`source_directories` value of `/etc` and `/var`—with `/home` omitted.
+
+This feature currently only works on scalar (e.g. string or number) list items
+and will not work elsewhere in a configuration file. Be sure to put the
+`!omit` tag *before* the list item (after the dash). Putting `!omit` after the
+list item will not work, as it gets interpreted as part of the string. Here's
+an example of some things not to do:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   source_directories:
+     # Do not do this! It will not work. "!omit" belongs before "/home".
+     - /home !omit
+
+   # Do not do this either! "!omit" only works on scalar list items.
+   repositories: !omit
+     # Also do not do this for the same reason! This is a list item, but it's
+     # not a scalar.
+     - !omit path: repo.borg
+```
+
+Additionally, the `!omit` tag only works in a configuration file that also
+performs a merge include with `<<: !include`. It doesn't make sense within,
+for instance, an included configuration file itself (unless it in turn
+performs its own merge include). That's because `!omit` only applies to the
+file doing the include; it doesn't work in reverse or propagate through
+includes.
+
+
+### Shallow merge
+
+Even though deep merging is generally pretty handy for included files,
+sometimes you want specific sections in the local file to take precedence over
+included sections—without any merging occurring for them.
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span> That's
+where the `!retain` tag comes in. Whenever you're merging an included file
+into your configuration file, you can optionally add the `!retain` tag to
+particular local mappings or lists to retain the local values and ignore
+included values.
+
+For instance, start with this configuration file containing the `!retain` tag
+on the `retention` mapping:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   repositories:
+     - path: repo.borg
+
+retention: !retain
+    keep_daily: 5
+```
+
+And `common.yaml` like this:
+
+```yaml
+location:
+   repositories:
+     - path: common.borg
+
+retention:
+    keep_hourly: 24
+    keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration will have a
+`keep_daily` value of `5` and nothing else in the `retention` section. That's
+because the `!retain` tag says to retain the local version of `retention` and
+ignore any values coming in from the include. But because the `repositories`
+list doesn't have a `!retain` tag, it still gets merged together to contain
+both `common.borg` and `repo.borg`.
+
+The `!retain` tag can only be placed on mappings and lists, and it goes right
+after the name of the option (and its colon) on the same line. The effects of
+`!retain` are recursive, meaning that if you place a `!retain` tag on a
+top-level mapping, even deeply nested values within it will not be merged.
+
+Additionally, the `!retain` tag only works in a configuration file that also
+performs a merge include with `<<: !include`. It doesn't make sense within,
+for instance, an included configuration file itself (unless it in turn
+performs its own merge include). That's because `!retain` only applies to the
+file doing the include; it doesn't work in reverse or propagate through
+includes.
+
+
+## Debugging includes
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span> If you'd
+like to see what the loaded configuration looks like after includes get merged
+in, run `validate-borgmatic-config` on your configuration file:
+
+```bash
+sudo validate-borgmatic-config --show
+```
+
+You'll need to specify your configuration file with `--config` if it's not in
+a default location.
+
+This will output the merged configuration as borgmatic sees it, which can be
+helpful for understanding how your includes work in practice.
+
 
 ## Configuration overrides
 

docs/how-to/set-up-backups.md

@@ -180,6 +180,9 @@ following command is available for that:
 sudo validate-borgmatic-config
 ```
 
+You'll need to specify your configuration file with `--config` if it's not in
+a default location.
+
 This command's exit status (`$?` in Bash) is zero when configuration is valid
 and non-zero otherwise.
 

scripts/run-end-to-end-dev-tests

@@ -10,5 +10,11 @@
 
 set -e
 
+USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock
+
+if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then
+    export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
+fi
+
 docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
     --renew-anon-volumes --abort-on-container-exit

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.7.11'
+VERSION = '1.7.12'
 
 
 setup(

tests/end-to-end/docker-compose.yaml

@@ -1,22 +1,22 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:13.1-alpine
+    image: docker.io/postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   mysql:
-    image: mariadb:10.5
+    image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   mongodb:
-    image: mongo:5.0.5
+    image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
   tests:
-    image: alpine:3.13
+    image: docker.io/alpine:3.13
     environment:
       TEST_CONTAINER: true
     volumes:

tests/end-to-end/test_validate_config.py

@@ -1,5 +1,6 @@
 import os
 import subprocess
+import sys
 import tempfile
 
 
@@ -26,3 +27,16 @@ def test_validate_config_command_with_invalid_configuration_fails():
         exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' '))
 
         assert exit_code == 1
+
+
+def test_validate_config_command_with_show_flag_displays_configuration():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        output = subprocess.check_output(
+            f'validate-borgmatic-config --config {config_path} --show'.split(' ')
+        ).decode(sys.stdout.encoding)
+
+        assert 'location:' in output
+        assert 'repositories:' in output

tests/integration/borg/test_commands.py

@@ -0,0 +1,108 @@
+import copy
+
+import flexmock
+
+import borgmatic.borg.info
+import borgmatic.borg.list
+import borgmatic.borg.rlist
+import borgmatic.borg.transfer
+import borgmatic.commands.arguments
+
+
+def assert_command_does_not_duplicate_flags(command, *args, **kwargs):
+    '''
+    Assert that the given Borg command sequence does not contain any duplicated flags, e.g.
+    "--match-archives" twice anywhere in the command.
+    '''
+    flag_counts = {}
+
+    for flag_name in command:
+        if not flag_name.startswith('--'):
+            continue
+
+        if flag_name in flag_counts:
+            flag_counts[flag_name] += 1
+        else:
+            flag_counts[flag_name] = 1
+
+    assert flag_counts == {
+        flag_name: 1 for flag_name in flag_counts
+    }, f"Duplicate flags found in: {' '.join(command)}"
+
+
+def fuzz_argument(arguments, argument_name):
+    '''
+    Given an argparse.Namespace instance of arguments and an argument name in it, copy the arguments
+    namespace and set the argument name in the copy with a fake value. Return the copied arguments.
+
+    This is useful for "fuzzing" a unit under test by passing it each possible argument in turn,
+    making sure it doesn't blow up or duplicate Borg arguments.
+    '''
+    arguments_copy = copy.copy(arguments)
+    value = getattr(arguments_copy, argument_name)
+    setattr(arguments_copy, argument_name, not value if isinstance(value, bool) else 'value')
+
+    return arguments_copy
+
+
+def test_transfer_archives_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments(
+        'transfer', '--source-repository', 'foo'
+    )['transfer']
+    flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with(
+        assert_command_does_not_duplicate_flags
+    )
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        borgmatic.borg.transfer.transfer_archives(
+            False, 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name)
+        )
+
+
+def test_make_list_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments('list')['list']
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        command = borgmatic.borg.list.make_list_command(
+            'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name)
+        )
+
+        assert_command_does_not_duplicate_flags(command)
+
+
+def test_make_rlist_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments('rlist')['rlist']
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        command = borgmatic.borg.rlist.make_rlist_command(
+            'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name)
+        )
+
+        assert_command_does_not_duplicate_flags(command)
+
+
+def test_display_archives_info_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments('info')['info']
+    flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with(
+        assert_command_does_not_duplicate_flags
+    )
+    flexmock(borgmatic.borg.info).should_receive('execute_command').replace_with(
+        assert_command_does_not_duplicate_flags
+    )
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        borgmatic.borg.info.display_archives_info(
+            'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name)
+        )

tests/integration/commands/test_arguments.py

@@ -465,6 +465,20 @@ def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives
         )
 
 
+def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar')
+
+
+def test_parse_arguments_disallows_rlist_with_both_prefix_and_match_archives():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('rlist', '--prefix', 'foo', '--match-archives', 'sh:*bar')
+
+
 def test_parse_arguments_disallows_info_with_both_archive_and_match_archives():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/commands/test_validate_config.py

@@ -18,3 +18,12 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list():
     parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
 
     assert parser.config_paths == ['myconfig', 'otherconfig']
+
+
+def test_parse_arguments_supports_show_flag():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments('--config', 'myconfig', '--show')
+
+    assert parser.show

tests/integration/config/test_load.py

@@ -2,7 +2,6 @@ import io
 import sys
 
 import pytest
-import ruamel.yaml
 from flexmock import flexmock
 
 from borgmatic.config import load as module
@@ -150,6 +149,248 @@ def test_load_configuration_merges_include():
     assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
 
 
+def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          foo: bar
+          baz: quux
+
+        other:
+          a: b
+          c: d
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff: !retain
+          foo: override
+
+        other:
+          a: override
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {
+        'stuff': {'foo': 'override'},
+        'other': {'a': 'override', 'c': 'd'},
+    }
+
+
+def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff: !retain
+          foo: bar
+          baz: quux
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          foo: override
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_retain_tag_on_scalar_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          foo: bar
+          baz: quux
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          foo: !retain override
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - !omit b
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']}
+
+
+def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - !omit q
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']}
+
+
+def test_load_configuration_with_omit_tag_on_non_list_item_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff: !omit
+          - x
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - foo: bar
+            baz: quux
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - !omit foo: bar
+            baz: quux
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_but_without_merge_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - !omit b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
 def test_load_configuration_does_not_merge_include_list():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -173,42 +414,79 @@ def test_load_configuration_does_not_merge_include_list():
     config_file.name = 'config.yaml'
     builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
 
-    with pytest.raises(ruamel.yaml.error.YAMLError):
+    with pytest.raises(module.ruamel.yaml.error.YAMLError):
         assert module.load_configuration('config.yaml')
 
 
+@pytest.mark.parametrize(
+    'node_class',
+    (
+        module.ruamel.yaml.nodes.MappingNode,
+        module.ruamel.yaml.nodes.SequenceNode,
+        module.ruamel.yaml.nodes.ScalarNode,
+    ),
+)
+def test_raise_retain_node_error_raises(node_class):
+    with pytest.raises(ValueError):
+        module.raise_retain_node_error(
+            loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock())
+        )
+
+
+def test_raise_omit_node_error_raises():
+    with pytest.raises(ValueError):
+        module.raise_omit_node_error(loader=flexmock(), node=flexmock())
+
+
+def test_filter_omitted_nodes():
+    nodes = [
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+    ]
+
+    result = module.filter_omitted_nodes(nodes)
+
+    assert [item.value for item in result] == ['a', 'c', 'a', 'c']
+
+
 def test_deep_merge_nodes_replaces_colliding_scalar_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_hourly'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
                     ),
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_daily'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
                     ),
                 ],
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_daily'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
                     ),
                 ],
             ),
@@ -230,35 +508,39 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values():
 def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_hourly'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
                     ),
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_daily'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
                     ),
                 ],
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_minutely'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='10'),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='10'
+                        ),
                     ),
                 ],
             ),
@@ -282,28 +564,28 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
 def test_deep_merge_nodes_keeps_deeply_nested_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='lock_wait'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
                     ),
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='extra_borg_options'
                         ),
-                        ruamel.yaml.nodes.MappingNode(
+                        module.ruamel.yaml.nodes.MappingNode(
                             tag='tag:yaml.org,2002:map',
                             value=[
                                 (
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='init'
                                     ),
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='--init-option'
                                     ),
                                 ),
@@ -314,22 +596,22 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='extra_borg_options'
                         ),
-                        ruamel.yaml.nodes.MappingNode(
+                        module.ruamel.yaml.nodes.MappingNode(
                             tag='tag:yaml.org,2002:map',
                             value=[
                                 (
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='prune'
                                     ),
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='--prune-option'
                                     ),
                                 ),
@@ -361,32 +643,48 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
 def test_deep_merge_nodes_appends_colliding_sequence_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
-                        ruamel.yaml.nodes.SequenceNode(
-                            tag='tag:yaml.org,2002:int', value=['echo 1', 'echo 2']
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
                         ),
                     ),
                 ],
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
-                        ruamel.yaml.nodes.SequenceNode(
-                            tag='tag:yaml.org,2002:int', value=['echo 3', 'echo 4']
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 4'
+                                ),
+                            ],
                         ),
                     ),
                 ],
@@ -401,4 +699,178 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
     options = section_value.value
     assert len(options) == 1
     assert options[0][0].value == 'before_backup'
-    assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
+    assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
+
+
+def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain():
+    node_values = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_hourly'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
+                    ),
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='!retain',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'retention'
+    assert section_value.tag == 'tag:yaml.org,2002:map'
+    options = section_value.value
+    assert len(options) == 1
+    assert options[0][0].value == 'keep_daily'
+    assert options[0][1].value == '5'
+
+
+def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain():
+    node_values = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='!retain',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 4'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'hooks'
+    options = section_value.value
+    assert len(options) == 1
+    assert options[0][0].value == 'before_backup'
+    assert options[0][1].tag == 'tag:yaml.org,2002:seq'
+    assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4']
+
+
+def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit():
+    node_values = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'hooks'
+    options = section_value.value
+    assert len(options) == 1
+    assert options[0][0].value == 'before_backup'
+    assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3']

tests/unit/borg/test_info.py

@@ -29,7 +29,7 @@ def test_display_archives_info_calls_borg_with_parameters():
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None),
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
     )
 
 
@@ -54,7 +54,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None),
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
     )
 
 
@@ -77,7 +77,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=True, prefix=None),
+        info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None),
     )
 
     assert json_output == '[]'
@@ -105,7 +105,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None),
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
     )
 
 
@@ -128,7 +128,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=True, prefix=None),
+        info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None),
     )
 
     assert json_output == '[]'
@@ -152,7 +152,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=True, prefix=None),
+        info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None),
     )
 
     assert json_output == '[]'
@@ -162,17 +162,14 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_flags').and_return(())
-    flexmock(module.flags).should_receive('make_flags').with_args(
-        'match-archives', 'archive'
-    ).and_return(('--match-archives', 'archive'))
     flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
-        None, None, '2.3.4'
-    ).and_return(())
+        'archive', None, '2.3.4'
+    ).and_return(('--match-archives', 'archive'))
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--repo', 'repo', '--match-archives', 'archive'),
+        ('borg', 'info', '--match-archives', 'archive', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         borg_local_path='borg',
         extra_environment=None,
@@ -182,7 +179,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive='archive', json=False, prefix=None),
+        info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None),
     )
 
 
@@ -207,7 +204,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path():
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None),
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
         local_path='borg1',
     )
 
@@ -236,7 +233,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None),
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
         remote_path='borg1',
     )
 
@@ -266,7 +263,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
         repository_path='repo',
         storage_config=storage_config,
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None),
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
     )
 
 
@@ -347,11 +344,64 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive
         repository_path='repo',
         storage_config={'archive_name_format': 'bar-{now}'},  # noqa: FS003
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None),
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
     )
 
 
-@pytest.mark.parametrize('argument_name', ('match_archives', 'sort_by', 'first', 'last'))
+def test_display_archives_with_match_archives_option_calls_borg_with_match_archives_parameter():
+    flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
+    flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
+    flexmock(module.flags).should_receive('make_flags').and_return(())
+    flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
+        'sh:foo-*', 'bar-{now}', '2.3.4'  # noqa: FS003
+    ).and_return(('--match-archives', 'sh:foo-*'))
+    flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
+    flexmock(module.environment).should_receive('make_environment')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'),
+        output_log_level=module.borgmatic.logger.ANSWER,
+        borg_local_path='borg',
+        extra_environment=None,
+    )
+
+    module.display_archives_info(
+        repository_path='repo',
+        storage_config={
+            'archive_name_format': 'bar-{now}',  # noqa: FS003
+            'match_archives': 'sh:foo-*',
+        },
+        local_borg_version='2.3.4',
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None),
+    )
+
+
+def test_display_archives_with_match_archives_flag_calls_borg_with_match_archives_parameter():
+    flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
+    flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
+    flexmock(module.flags).should_receive('make_flags').and_return(())
+    flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
+        'sh:foo-*', 'bar-{now}', '2.3.4'  # noqa: FS003
+    ).and_return(('--match-archives', 'sh:foo-*'))
+    flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
+    flexmock(module.environment).should_receive('make_environment')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'info', '--match-archives', 'sh:foo-*', '--repo', 'repo'),
+        output_log_level=module.borgmatic.logger.ANSWER,
+        borg_local_path='borg',
+        extra_environment=None,
+    )
+
+    module.display_archives_info(
+        repository_path='repo',
+        storage_config={'archive_name_format': 'bar-{now}'},  # noqa: FS003
+        local_borg_version='2.3.4',
+        info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'),
+    )
+
+
+@pytest.mark.parametrize('argument_name', ('sort_by', 'first', 'last'))
 def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
@@ -376,5 +426,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
         repository_path='repo',
         storage_config={},
         local_borg_version='2.3.4',
-        info_arguments=flexmock(archive=None, json=False, prefix=None, **{argument_name: 'value'}),
+        info_arguments=flexmock(
+            archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'}
+        ),
     )

tests/unit/borg/test_prune.py

@@ -69,6 +69,24 @@ def test_make_prune_flags_with_prefix_without_borg_features_uses_glob_archives()
     assert result == expected
 
 
+def test_make_prune_flags_prefers_prefix_to_archive_name_format():
+    storage_config = {'archive_name_format': 'bar-{now}'}  # noqa: FS003
+    retention_config = OrderedDict((('keep_daily', 1), ('prefix', 'bar-')))
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.flags).should_receive('make_match_archives_flags').never()
+
+    result = module.make_prune_flags(storage_config, retention_config, local_borg_version='1.2.3')
+
+    expected = (
+        '--keep-daily',
+        '1',
+        '--match-archives',
+        'sh:bar-*',  # noqa: FS003
+    )
+
+    assert result == expected
+
+
 def test_make_prune_flags_without_prefix_uses_archive_name_format_instead():
     storage_config = {'archive_name_format': 'bar-{now}'}  # noqa: FS003
     retention_config = OrderedDict((('keep_daily', 1), ('prefix', None)))

tests/unit/borg/test_rlist.py

@@ -137,7 +137,9 @@ def test_make_rlist_command_includes_log_info():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=False, prefix=None, match_archives=None
+        ),
     )
 
     assert command == ('borg', 'list', '--info', 'repo')
@@ -156,7 +158,9 @@ def test_make_rlist_command_includes_json_but_not_info():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=True, prefix=None, match_archives=None
+        ),
     )
 
     assert command == ('borg', 'list', '--json', 'repo')
@@ -175,7 +179,9 @@ def test_make_rlist_command_includes_log_debug():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=False, prefix=None, match_archives=None
+        ),
     )
 
     assert command == ('borg', 'list', '--debug', '--show-rc', 'repo')
@@ -194,7 +200,9 @@ def test_make_rlist_command_includes_json_but_not_debug():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=True, prefix=None, match_archives=None
+        ),
     )
 
     assert command == ('borg', 'list', '--json', 'repo')
@@ -212,7 +220,9 @@ def test_make_rlist_command_includes_json():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=True, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=True, prefix=None, match_archives=None
+        ),
     )
 
     assert command == ('borg', 'list', '--json', 'repo')
@@ -232,7 +242,9 @@ def test_make_rlist_command_includes_lock_wait():
         repository_path='repo',
         storage_config={'lock_wait': 5},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=False, prefix=None, match_archives=None
+        ),
     )
 
     assert command == ('borg', 'list', '--lock-wait', '5', 'repo')
@@ -250,7 +262,9 @@ def test_make_rlist_command_includes_local_path():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=False, prefix=None, match_archives=None
+        ),
         local_path='borg2',
     )
 
@@ -271,7 +285,9 @@ def test_make_rlist_command_includes_remote_path():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=False, prefix=None, match_archives=None
+        ),
         remote_path='borg2',
     )
 
@@ -328,7 +344,9 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives()
         repository_path='repo',
         storage_config={'archive_name_format': 'bar-{now}'},  # noqa: FS003
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=False, prefix=None, match_archives=None
+        ),
     )
 
     assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo')
@@ -346,7 +364,9 @@ def test_make_rlist_command_includes_short():
         repository_path='repo',
         storage_config={},
         local_borg_version='1.2.3',
-        rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix=None, short=True),
+        rlist_arguments=flexmock(
+            archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True
+        ),
     )
 
     assert command == ('borg', 'list', '--short', 'repo')
@@ -354,16 +374,7 @@ def test_make_rlist_command_includes_short():
 
 @pytest.mark.parametrize(
     'argument_name',
-    (
-        'match_archives',
-        'sort_by',
-        'first',
-        'last',
-        'exclude',
-        'exclude_from',
-        'pattern',
-        'patterns_from',
-    ),
+    ('sort_by', 'first', 'last', 'exclude', 'exclude_from', 'pattern', 'patterns_from',),
 )
 def test_make_rlist_command_includes_additional_flags(argument_name):
     flexmock(module.flags).should_receive('make_flags').and_return(())
@@ -384,6 +395,7 @@ def test_make_rlist_command_includes_additional_flags(argument_name):
             paths=None,
             json=False,
             prefix=None,
+            match_archives=None,
             find_paths=None,
             format=None,
             **{argument_name: 'value'},
@@ -393,6 +405,35 @@ def test_make_rlist_command_includes_additional_flags(argument_name):
     assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo')
 
 
+def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_parameters():
+    flexmock(module.flags).should_receive('make_flags').and_return(())
+    flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
+        None, None, '1.2.3'
+    ).and_return(())
+    flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
+        'foo-*', None, '1.2.3',
+    ).and_return(('--match-archives', 'foo-*'))
+    flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+
+    command = module.make_rlist_command(
+        repository_path='repo',
+        storage_config={},
+        local_borg_version='1.2.3',
+        rlist_arguments=flexmock(
+            archive=None,
+            paths=None,
+            json=False,
+            prefix=None,
+            match_archives='foo-*',
+            find_paths=None,
+            format=None,
+        ),
+    )
+
+    assert command == ('borg', 'list', '--match-archives', 'foo-*', 'repo')
+
+
 def test_list_repository_calls_borg_with_parameters():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER

tests/unit/borg/test_transfer.py

@@ -124,10 +124,9 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_flags').and_return(())
-    flexmock(module.flags).should_receive('make_flags').with_args(
-        'match-archives', 'archive'
+    flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
+        'archive', 'bar-{now}', '2.3.4'  # noqa: FS003
     ).and_return(('--match-archives', 'archive'))
-    flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
@@ -154,10 +153,9 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_flags').and_return(())
-    flexmock(module.flags).should_receive('make_flags').with_args(
-        'match-archives', 'sh:foo*'
+    flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
+        'sh:foo*', 'bar-{now}', '2.3.4'  # noqa: FS003
     ).and_return(('--match-archives', 'sh:foo*'))
-    flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
@@ -304,7 +302,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag():
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_flags').and_return(())
     flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
-    flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
+    flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',))
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(

tests/unit/commands/test_borgmatic.py

@@ -408,7 +408,7 @@ def test_run_actions_runs_rcreate():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'rcreate': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rcreate': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -423,6 +423,43 @@ def test_run_actions_runs_rcreate():
     )
 
 
+def test_run_actions_adds_log_file_to_hook_context():
+    flexmock(module).should_receive('add_custom_log_levels')
+    flexmock(module.command).should_receive('execute_hook')
+    expected = flexmock()
+    flexmock(borgmatic.actions.create).should_receive('run_create').with_args(
+        config_filename=object,
+        repository={'path': 'repo'},
+        location={'repositories': []},
+        storage=object,
+        hooks={},
+        hook_context={'repository': 'repo', 'repositories': '', 'log_file': 'foo'},
+        local_borg_version=object,
+        create_arguments=object,
+        global_arguments=object,
+        dry_run_label='',
+        local_path=object,
+        remote_path=object,
+    ).once().and_return(expected)
+
+    result = tuple(
+        module.run_actions(
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()},
+            config_filename=flexmock(),
+            location={'repositories': []},
+            storage=flexmock(),
+            retention=flexmock(),
+            consistency=flexmock(),
+            hooks={},
+            local_path=flexmock(),
+            remote_path=flexmock(),
+            local_borg_version=flexmock(),
+            repository={'path': 'repo'},
+        )
+    )
+    assert result == (expected,)
+
+
 def test_run_actions_runs_transfer():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module.command).should_receive('execute_hook')
@@ -430,7 +467,7 @@ def test_run_actions_runs_transfer():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'transfer': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'transfer': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -453,7 +490,7 @@ def test_run_actions_runs_create():
 
     result = tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'create': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -476,7 +513,7 @@ def test_run_actions_runs_prune():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'prune': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -498,7 +535,7 @@ def test_run_actions_runs_compact():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'compact': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -521,7 +558,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'check': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -544,7 +581,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'check': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -566,7 +603,7 @@ def test_run_actions_runs_extract():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'extract': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'extract': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -588,7 +625,7 @@ def test_run_actions_runs_export_tar():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'export-tar': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export-tar': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -610,7 +647,7 @@ def test_run_actions_runs_mount():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'mount': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'mount': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -632,7 +669,7 @@ def test_run_actions_runs_restore():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'restore': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'restore': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -655,7 +692,7 @@ def test_run_actions_runs_rlist():
 
     result = tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'rlist': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rlist': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -679,7 +716,7 @@ def test_run_actions_runs_list():
 
     result = tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'list': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'list': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -703,7 +740,7 @@ def test_run_actions_runs_rinfo():
 
     result = tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'rinfo': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'rinfo': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -727,7 +764,7 @@ def test_run_actions_runs_info():
 
     result = tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'info': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'info': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -750,7 +787,7 @@ def test_run_actions_runs_break_lock():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'break-lock': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'break-lock': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -772,7 +809,7 @@ def test_run_actions_runs_borg():
 
     tuple(
         module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'borg': flexmock()},
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'borg': flexmock()},
             config_filename=flexmock(),
             location={'repositories': []},
             storage=flexmock(),
@@ -796,7 +833,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order():
     tuple(
         module.run_actions(
             arguments={
-                'global': flexmock(dry_run=False),
+                'global': flexmock(dry_run=False, log_file='foo'),
                 'borg': flexmock(),
                 'restore': flexmock(),
             },