Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/696

.drone.yml

@@ -1,19 +1,20 @@
+---
 kind: pipeline
 name: python-3-8-alpine-3-13
 
 services:
   - name: postgresql
-    image: postgres:13.1-alpine
+    image: docker.io/postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   - name: mysql
-    image: mariadb:10.5
+    image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   - name: mongodb
-    image: mongo:5.0.5
+    image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
@@ -23,34 +24,41 @@ clone:
 
 steps:
 - name: build
-  image: alpine:3.13
+  image: docker.io/alpine:3.13
+  environment:
+    TEST_CONTAINER: true
   pull: always
   commands:
     - scripts/run-full-tests
 ---
 kind: pipeline
 name: documentation
+type: exec
+
+platform:
+  os: linux
+  arch: amd64
 
 clone:
   skip_verify: true
 
 steps:
 - name: build
-  image: plugins/docker
-  settings:
-    username:
+  environment:
+    USERNAME:
       from_secret: docker_username
-    password:
+    PASSWORD:
       from_secret: docker_password
-    registry: projects.torsion.org
-    repo: projects.torsion.org/borgmatic-collective/borgmatic
-    tags: docs
-    dockerfile: docs/Dockerfile
+    IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
+  commands:
+    - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
+    - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
+    - podman push "$IMAGE_NAME"
 
 trigger:
   repo:
     - borgmatic-collective/borgmatic
   branch:
-    - master
+    - main
   event:
     - push

NEWS

@@ -1,3 +1,66 @@
+1.7.13
+ * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema"
+   flag. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas
+ * #678: Fix error from PostgreSQL when dumping a database with a "format" of "plain".
+ * #678: Fix PostgreSQL hook to support "psql_command" and "pg_restore_command" options containing
+   commands with arguments.
+ * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break
+   database dumping.
+ * #680: Add support for logging each log line as a JSON object via global "--log-json" flag.
+ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories.
+ * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to
+   update your development checkouts accordingly.
+ * #686: Add fish shell completion script so you can tab-complete on the borgmatic command-line. See
+   the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion
+ * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install
+   --editable" development installs.
+ * #688: Fix archive checks being skipped even when particular archives haven't been checked
+   recently. This occurred when using multiple borgmatic configuration files with different
+   "archive_name_format"s, for instance.
+ * #691: Fix error in "borgmatic restore" action when the configured repository path is relative
+   instead of absolute.
+ * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like
+   "--progress" still work.
+
+1.7.12
+ * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #666, #670: Fix error when running the "info" action with the "--match-archives" or "--archive"
+   flags. Also fix the "--match-archives"/"--archive" flags to correctly override the
+   "match_archives" configuration option for the "transfer", "list", "rlist", and "info" actions.
+ * #668: Fix error when running the "prune" action with both "archive_name_format" and "prefix"
+   options set.
+ * #672: Selectively shallow merge certain mappings or sequences when including configuration files.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#shallow-merge
+ * #672: Selectively omit list values when including configuration files. See the documentation for
+   more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#list-merge
+ * #673: View the results of configuration file merging via "validate-borgmatic-config --show" flag.
+   See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#debugging-includes
+ * Add optional support for running end-to-end tests and building documentation with rootless Podman
+   instead of Docker.
+
+1.7.11
+ * #479, #588: BREAKING: Automatically use the "archive_name_format" option to filter which archives
+   get used for borgmatic actions that operate on multiple archives. Override this behavior with the
+   new "match_archives" option in the storage section. This change is "breaking" in that it silently
+   changes which archives get considered for "rlist", "prune", "check", etc. See the documentation
+   for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#archive-naming
+ * #479, #588: The "prefix" options have been deprecated in favor of the new "archive_name_format"
+   auto-matching behavior and the "match_archives" option.
+ * #658: Add "--log-file-format" flag for customizing the log message format. See the documentation
+   for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#logging-to-file
+ * #662: Fix regression in which the "check_repositories" option failed to match repositories.
+ * #663: Fix regression in which the "transfer" action produced a traceback.
+ * Add spellchecking of source code during test runs.
+
 1.7.10
  * #396: When a database command errors, display and log the error message instead of swallowing it.
  * #501: Optionally error if a source directory does not exist via "source_directories_must_exist"
@@ -325,7 +388,7 @@
  * #398: Clarify canonical home of borgmatic in documentation.
  * #406: Clarify that spaces in path names should not be backslashed in path names.
  * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory!
- * Fix build so as not to attempt to build and push documentation for a non-master branch.
+ * Fix build so as not to attempt to build and push documentation for a non-main branch.
  * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13.
  * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama.
    IRC connection info: https://torsion.org/borgmatic/#issues
@@ -388,7 +451,7 @@
    configuration schema descriptions.
 
 1.5.6
- * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
+ * #292: Allow before_backup and similar hooks to exit with a soft failure without altering the
    monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring
    services with a "start" status until after before_* hooks finish. Failures in before_* hooks
    still trigger a monitoring "fail" status.
@@ -457,7 +520,7 @@
  * For "list" and "info" actions, show repository names even at verbosity 0.
 
 1.4.22
- * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON ouput.
+ * #276, #285: Disable colored output when "--json" flag is used, so as to produce valid JSON output.
  * After a backup of a database dump in directory format, properly remove the dump directory.
  * In "borgmatic --help", don't expand $HOME in listing of default "--config" paths.
 
@@ -829,7 +892,7 @@
  * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
    editor swap files, etc.
  * #81: Document user-defined hooks run before/after backup, or on error.
- * Add code style guidelines to the documention.
+ * Add code style guidelines to the documentation.
 
 1.2.0
  * #61: Support for Borg --list option via borgmatic command-line to list all archives.

README.md

@@ -11,7 +11,7 @@ borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
 databases too. Monitor it all with integrated third-party services.
 
-The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
+The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic/</a>
 
 Here's an example configuration file:
 
@@ -165,5 +165,5 @@ Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
-<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master)</a>
+<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/main)</a>
 

SECURITY.md

@@ -7,8 +7,8 @@ permalink: security-policy/index.html
 
 While we want to hear about security vulnerabilities in all versions of
 borgmatic, security fixes are only made to the most recently released version.
-It's simply not practical for our small volunteer effort to maintain multiple
-release branches and put out separate security patches for each.
+It's not practical for our small volunteer effort to maintain multiple release
+branches and put out separate security patches for each.
 
 ## Reporting a vulnerability
 

borgmatic/actions/borg.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_borg(
-    repository, storage, local_borg_version, borg_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    borg_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "borg" action for the given repository.
@@ -16,12 +22,15 @@ def run_borg(
     if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, borg_arguments.repository
     ):
-        logger.info(f'{repository["path"]}: Running arbitrary Borg command')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Running arbitrary Borg command'
+        )
         archive_name = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             borg_arguments.archive,
             storage,
             local_borg_version,
+            global_arguments,
             local_path,
             remote_path,
         )

borgmatic/actions/break_lock.py

@@ -7,7 +7,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_break_lock(
-    repository, storage, local_borg_version, break_lock_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    break_lock_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "break-lock" action for the given repository.
@@ -15,11 +21,14 @@ def run_break_lock(
     if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, break_lock_arguments.repository
     ):
-        logger.info(f'{repository["path"]}: Breaking repository and cache locks')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Breaking repository and cache locks'
+        )
         borgmatic.borg.break_lock.break_lock(
             repository['path'],
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/check.py

@@ -37,13 +37,14 @@ def run_check(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository["path"]}: Running consistency checks')
+    logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
     borgmatic.borg.check.check_archives(
         repository['path'],
         location,
         storage,
         consistency,
         local_borg_version,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
         progress=check_arguments.progress,

borgmatic/actions/compact.py

@@ -39,12 +39,15 @@ def run_compact(
         **hook_context,
     )
     if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
-        logger.info(f'{repository["path"]}: Compacting segments{dry_run_label}')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}'
+        )
         borgmatic.borg.compact.compact_segments(
             global_arguments.dry_run,
             repository['path'],
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
             progress=compact_arguments.progress,
@@ -52,7 +55,9 @@ def run_compact(
             threshold=compact_arguments.threshold,
         )
     else:  # pragma: nocover
-        logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)'
+        )
     borgmatic.hooks.command.execute_hook(
         hooks.get('after_compact'),
         hooks.get('umask'),

borgmatic/actions/create.py

@@ -42,7 +42,7 @@ def run_create(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository["path"]}: Creating archive{dry_run_label}')
+    logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
         'remove_database_dumps',
         hooks,
@@ -67,6 +67,7 @@ def run_create(
         location,
         storage,
         local_borg_version,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
         progress=create_arguments.progress,

borgmatic/actions/export_tar.py

@@ -33,6 +33,7 @@ def run_export_tar(
                 export_tar_arguments.archive,
                 storage,
                 local_borg_version,
+                global_arguments,
                 local_path,
                 remote_path,
             ),
@@ -40,6 +41,7 @@ def run_export_tar(
             export_tar_arguments.destination,
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
             tar_filter=export_tar_arguments.tar_filter,

borgmatic/actions/extract.py

@@ -35,7 +35,9 @@ def run_extract(
     if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, extract_arguments.repository
     ):
-        logger.info(f'{repository["path"]}: Extracting archive {extract_arguments.archive}')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}'
+        )
         borgmatic.borg.extract.extract_archive(
             global_arguments.dry_run,
             repository['path'],
@@ -44,6 +46,7 @@ def run_extract(
                 extract_arguments.archive,
                 storage,
                 local_borg_version,
+                global_arguments,
                 local_path,
                 remote_path,
             ),
@@ -51,6 +54,7 @@ def run_extract(
             location,
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
             destination_path=extract_arguments.destination,

borgmatic/actions/info.py

@@ -9,7 +9,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_info(
-    repository, storage, local_borg_version, info_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    info_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "info" action for the given repository and archive.
@@ -20,12 +26,15 @@ def run_info(
         repository, info_arguments.repository
     ):
         if not info_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository["path"]}: Displaying archive summary information')
+            logger.answer(
+                f'{repository.get("label", repository["path"])}: Displaying archive summary information'
+            )
         info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             info_arguments.archive,
             storage,
             local_borg_version,
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -33,9 +42,10 @@ def run_info(
             repository['path'],
             storage,
             local_borg_version,
-            info_arguments=info_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
+            info_arguments,
+            global_arguments,
+            local_path,
+            remote_path,
         )
         if json_output:  # pragma: nocover
             yield json.loads(json_output)

borgmatic/actions/list.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_list(
-    repository, storage, local_borg_version, list_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    list_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "list" action for the given repository and archive.
@@ -20,14 +26,15 @@ def run_list(
     ):
         if not list_arguments.json:  # pragma: nocover
             if list_arguments.find_paths:
-                logger.answer(f'{repository["path"]}: Searching archives')
+                logger.answer(f'{repository.get("label", repository["path"])}: Searching archives')
             elif not list_arguments.archive:
-                logger.answer(f'{repository["path"]}: Listing archives')
+                logger.answer(f'{repository.get("label", repository["path"])}: Listing archives')
         list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             list_arguments.archive,
             storage,
             local_borg_version,
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -35,9 +42,10 @@ def run_list(
             repository['path'],
             storage,
             local_borg_version,
-            list_arguments=list_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
+            list_arguments,
+            global_arguments,
+            local_path,
+            remote_path,
         )
         if json_output:  # pragma: nocover
             yield json.loads(json_output)

borgmatic/actions/mount.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_mount(
-    repository, storage, local_borg_version, mount_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    mount_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "mount" action for the given repository.
@@ -17,9 +23,11 @@ def run_mount(
         repository, mount_arguments.repository
     ):
         if mount_arguments.archive:
-            logger.info(f'{repository["path"]}: Mounting archive {mount_arguments.archive}')
+            logger.info(
+                f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}'
+            )
         else:  # pragma: nocover
-            logger.info(f'{repository["path"]}: Mounting repository')
+            logger.info(f'{repository.get("label", repository["path"])}: Mounting repository')
 
         borgmatic.borg.mount.mount_archive(
             repository['path'],
@@ -28,6 +36,7 @@ def run_mount(
                 mount_arguments.archive,
                 storage,
                 local_borg_version,
+                global_arguments,
                 local_path,
                 remote_path,
             ),
@@ -37,6 +46,7 @@ def run_mount(
             mount_arguments.options,
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/prune.py

@@ -37,13 +37,14 @@ def run_prune(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository["path"]}: Pruning archives{dry_run_label}')
+    logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}')
     borgmatic.borg.prune.prune_archives(
         global_arguments.dry_run,
         repository['path'],
         storage,
         retention,
         local_borg_version,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
         stats=prune_arguments.stats,

borgmatic/actions/rcreate.py

@@ -23,12 +23,13 @@ def run_rcreate(
     ):
         return
 
-    logger.info(f'{repository["path"]}: Creating repository')
+    logger.info(f'{repository.get("label", repository["path"])}: Creating repository')
     borgmatic.borg.rcreate.create_repository(
         global_arguments.dry_run,
         repository['path'],
         storage,
         local_borg_version,
+        global_arguments,
         rcreate_arguments.encryption_mode,
         rcreate_arguments.source_repository,
         rcreate_arguments.copy_crypt_key,

borgmatic/actions/restore.py

@@ -73,12 +73,14 @@ def restore_single_database(
     Given (among other things) an archive name, a database hook name, and a configured database
     configuration dict, restore that database from the archive.
     '''
-    logger.info(f'{repository}: Restoring database {database["name"]}')
+    logger.info(
+        f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}'
+    )
 
     dump_pattern = borgmatic.hooks.dispatch.call_hooks(
         'make_database_dump_pattern',
         hooks,
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         database['name'],
@@ -87,12 +89,13 @@ def restore_single_database(
     # Kick off a single database extract to stdout.
     extract_process = borgmatic.borg.extract.extract_archive(
         dry_run=global_arguments.dry_run,
-        repository=repository,
+        repository=repository['path'],
         archive=archive_name,
         paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
         location_config=location,
         storage_config=storage,
         local_borg_version=local_borg_version,
+        global_arguments=global_arguments,
         local_path=local_path,
         remote_path=remote_path,
         destination_path='/',
@@ -105,7 +108,7 @@ def restore_single_database(
     borgmatic.hooks.dispatch.call_hooks(
         'restore_database_dump',
         {hook_name: [database]},
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         global_arguments.dry_run,
@@ -114,13 +117,20 @@ def restore_single_database(
 
 
 def collect_archive_database_names(
-    repository, archive, location, storage, local_borg_version, local_path, remote_path,
+    repository,
+    archive,
+    location,
+    storage,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Given a local or remote repository path, a resolved archive name, a location configuration dict,
-    a storage configuration dict, the local Borg version, and local and remote Borg paths, query the
-    archive for the names of databases it contains and return them as a dict from hook name to a
-    sequence of database names.
+    a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace,
+    and local and remote Borg paths, query the archive for the names of databases it contains and
+    return them as a dict from hook name to a sequence of database names.
     '''
     borgmatic_source_directory = os.path.expanduser(
         location.get(
@@ -135,6 +145,7 @@ def collect_archive_database_names(
         archive,
         storage,
         local_borg_version,
+        global_arguments,
         list_path=parent_dump_path,
         local_path=local_path,
         remote_path=remote_path,
@@ -180,7 +191,7 @@ def find_databases_to_restore(requested_database_names, archive_database_names):
     if 'all' in restore_names[UNSPECIFIED_HOOK]:
         restore_names[UNSPECIFIED_HOOK].remove('all')
 
-        for (hook_name, database_names) in archive_database_names.items():
+        for hook_name, database_names in archive_database_names.items():
             restore_names.setdefault(hook_name, []).extend(database_names)
 
             # If a database is to be restored as part of "all", then remove it from restore names so
@@ -256,7 +267,7 @@ def run_restore(
         return
 
     logger.info(
-        f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}'
+        f'{repository.get("label", repository["path"])}: Restoring databases from archive {restore_arguments.archive}'
     )
 
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
@@ -273,6 +284,7 @@ def run_restore(
         restore_arguments.archive,
         storage,
         local_borg_version,
+        global_arguments,
         local_path,
         remote_path,
     )
@@ -282,6 +294,7 @@ def run_restore(
         location,
         storage,
         local_borg_version,
+        global_arguments,
         local_path,
         remote_path,
     )
@@ -303,7 +316,7 @@ def run_restore(
 
             found_names.add(database_name)
             restore_single_database(
-                repository['path'],
+                repository,
                 location,
                 storage,
                 hooks,
@@ -313,7 +326,7 @@ def run_restore(
                 remote_path,
                 archive_name,
                 found_hook_name or hook_name,
-                found_database,
+                dict(found_database, **{'schemas': restore_arguments.schemas}),
             )
 
     # For any database that weren't found via exact matches in the hooks configuration, try to
@@ -332,7 +345,7 @@ def run_restore(
             database['name'] = database_name
 
             restore_single_database(
-                repository['path'],
+                repository,
                 location,
                 storage,
                 hooks,
@@ -342,7 +355,7 @@ def run_restore(
                 remote_path,
                 archive_name,
                 found_hook_name or hook_name,
-                database,
+                dict(database, **{'schemas': restore_arguments.schemas}),
             )
 
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(

borgmatic/actions/rinfo.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_rinfo(
-    repository, storage, local_borg_version, rinfo_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    rinfo_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "rinfo" action for the given repository.
@@ -19,13 +25,16 @@ def run_rinfo(
         repository, rinfo_arguments.repository
     ):
         if not rinfo_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository["path"]}: Displaying repository summary information')
+            logger.answer(
+                f'{repository.get("label", repository["path"])}: Displaying repository summary information'
+            )
 
         json_output = borgmatic.borg.rinfo.display_repository_info(
             repository['path'],
             storage,
             local_borg_version,
             rinfo_arguments=rinfo_arguments,
+            global_arguments=global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/rlist.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_rlist(
-    repository, storage, local_borg_version, rlist_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    rlist_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "rlist" action for the given repository.
@@ -19,13 +25,14 @@ def run_rlist(
         repository, rlist_arguments.repository
     ):
         if not rlist_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository["path"]}: Listing repository')
+            logger.answer(f'{repository.get("label", repository["path"])}: Listing repository')
 
         json_output = borgmatic.borg.rlist.list_repository(
             repository['path'],
             storage,
             local_borg_version,
             rlist_arguments=rlist_arguments,
+            global_arguments=global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/transfer.py

@@ -17,13 +17,16 @@ def run_transfer(
     '''
     Run the "transfer" action for the given repository.
     '''
-    logger.info(f'{repository}: Transferring archives to repository')
+    logger.info(
+        f'{repository.get("label", repository["path"])}: Transferring archives to repository'
+    )
     borgmatic.borg.transfer.transfer_archives(
         global_arguments.dry_run,
-        repository,
+        repository['path'],
         storage,
         local_borg_version,
         transfer_arguments,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
     )

borgmatic/borg/borg.py

@@ -1,8 +1,9 @@
 import logging
 
+import borgmatic.commands.arguments
 import borgmatic.logger
 from borgmatic.borg import environment, flags
-from borgmatic.execute import execute_command
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +37,14 @@ def run_arbitrary_borg(
         command_options_start_index = 2 if options[0] in BORG_SUBCOMMANDS_WITH_SUBCOMMANDS else 1
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
+
+        if (
+            borg_command
+            and borg_command[0] in borgmatic.commands.arguments.SUBPARSER_ALIASES.keys()
+        ):
+            logger.warning(
+                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
+            )
     except IndexError:
         borg_command = ()
         command_options = ()
@@ -62,7 +71,7 @@ def run_arbitrary_borg(
 
     return execute_command(
         full_command,
-        output_log_level=logging.ANSWER,
+        output_file=DO_NOT_CAPTURE,
         borg_local_path=local_path,
         extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/break_lock.py

@@ -7,12 +7,17 @@ logger = logging.getLogger(__name__)
 
 
 def break_lock(
-    repository_path, storage_config, local_borg_version, local_path='borg', remote_path=None,
+    repository_path,
+    storage_config,
+    local_borg_version,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
 ):
     '''
     Given a local or remote repository path, a storage configuration dict, the local Borg version,
-    and optional local and remote Borg paths, break any repository and cache locks leftover from Borg
-    aborting.
+    an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
+    repository and cache locks leftover from Borg aborting.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
@@ -21,6 +26,7 @@ def break_lock(
         (local_path, 'break-lock')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())

borgmatic/borg/check.py

@@ -1,5 +1,7 @@
 import argparse
 import datetime
+import hashlib
+import itertools
 import json
 import logging
 import os
@@ -12,7 +14,6 @@ DEFAULT_CHECKS = (
     {'name': 'repository', 'frequency': '1 month'},
     {'name': 'archives', 'frequency': '1 month'},
 )
-DEFAULT_PREFIX = '{hostname}-'  # noqa: FS003
 
 
 logger = logging.getLogger(__name__)
@@ -89,12 +90,18 @@ def parse_frequency(frequency):
 
 
 def filter_checks_on_frequency(
-    location_config, consistency_config, borg_repository_id, checks, force
+    location_config,
+    consistency_config,
+    borg_repository_id,
+    checks,
+    force,
+    archives_check_id=None,
 ):
     '''
     Given a location config, a consistency config with a "checks" sequence of dicts, a Borg
-    repository ID, a sequence of checks, and whether to force checks to run, filter down those
-    checks based on the configured "frequency" for each check as compared to its check time file.
+    repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives
+    check potentially being run (if any), filter down those checks based on the configured
+    "frequency" for each check as compared to its check time file.
 
     In other words, a check whose check time file's timestamp is too new (based on the configured
     frequency) will get cut from the returned sequence of checks. Example:
@@ -128,8 +135,8 @@ def filter_checks_on_frequency(
         if not frequency_delta:
             continue
 
-        check_time = read_check_time(
-            make_check_time_path(location_config, borg_repository_id, check)
+        check_time = probe_for_check_time(
+            location_config, borg_repository_id, check, archives_check_id
         )
         if not check_time:
             continue
@@ -146,10 +153,61 @@ def filter_checks_on_frequency(
     return tuple(filtered_checks)
 
 
-def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
+def make_archive_filter_flags(
+    local_borg_version, storage_config, checks, check_last=None, prefix=None
+):
     '''
-    Given the local Borg version and a parsed sequence of checks, transform the checks into tuple of
-    command-line flags.
+    Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the
+    check last value, and a consistency check prefix, transform the checks into tuple of
+    command-line flags for filtering archives in a check command.
+
+    If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if
+    a prefix value is given and "archives" is in checks, then include a "--match-archives" flag.
+    '''
+    if 'archives' in checks or 'data' in checks:
+        return (('--last', str(check_last)) if check_last else ()) + (
+            (
+                ('--match-archives', f'sh:{prefix}*')
+                if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
+                else ('--glob-archives', f'{prefix}*')
+            )
+            if prefix
+            else (
+                flags.make_match_archives_flags(
+                    storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
+        )
+
+    if check_last:
+        logger.warning(
+            'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
+        )
+    if prefix:
+        logger.warning(
+            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
+        )
+
+    return ()
+
+
+def make_archives_check_id(archive_filter_flags):
+    '''
+    Given a sequence of flags to filter archives, return a unique hash corresponding to those
+    particular flags. If there are no flags, return None.
+    '''
+    if not archive_filter_flags:
+        return None
+
+    return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
+
+
+def make_check_flags(checks, archive_filter_flags):
+    '''
+    Given a parsed sequence of checks and a sequence of flags to filter archives, transform the
+    checks into tuple of command-line check flags.
 
     For example, given parsed checks of:
 
@@ -161,10 +219,6 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
 
     However, if both "repository" and "archives" are in checks, then omit them from the returned
     flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
-
-    Additionally, if a check_last value is given and "archives" is in checks, then include a
-    "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
-    "--match-archives" flag.
     '''
     if 'data' in checks:
         data_flags = ('--verify-data',)
@@ -172,25 +226,7 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
     else:
         data_flags = ()
 
-    if 'archives' in checks:
-        last_flags = ('--last', str(check_last)) if check_last else ()
-        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-            match_archives_flags = ('--match-archives', f'sh:{prefix}*') if prefix else ()
-        else:
-            match_archives_flags = ('--glob-archives', f'{prefix}*') if prefix else ()
-    else:
-        last_flags = ()
-        match_archives_flags = ()
-        if check_last:
-            logger.warning(
-                'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
-            )
-        if prefix:
-            logger.warning(
-                'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
-            )
-
-    common_flags = last_flags + match_archives_flags + data_flags
+    common_flags = archive_filter_flags + data_flags
 
     if {'repository', 'archives'}.issubset(set(checks)):
         return common_flags
@@ -201,18 +237,27 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
     )
 
 
-def make_check_time_path(location_config, borg_repository_id, check_type):
+def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None):
     '''
-    Given a location configuration dict, a Borg repository ID, and the name of a check type
-    ("repository", "archives", etc.), return a path for recording that check's time (the time of
-    that check last occurring).
+    Given a location configuration dict, a Borg repository ID, the name of a check type
+    ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
+    path for recording that check's time (the time of that check last occurring).
     '''
+    borgmatic_source_directory = os.path.expanduser(
+        location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY)
+    )
+
+    if check_type in ('archives', 'data'):
+        return os.path.join(
+            borgmatic_source_directory,
+            'checks',
+            borg_repository_id,
+            check_type,
+            archives_check_id if archives_check_id else 'all',
+        )
+
     return os.path.join(
-        os.path.expanduser(
-            location_config.get(
-                'borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
-            )
-        ),
+        borgmatic_source_directory,
         'checks',
         borg_repository_id,
         check_type,
@@ -242,12 +287,81 @@ def read_check_time(path):
         return None
 
 
+def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id):
+    '''
+    Given a location configuration dict, a Borg repository ID, the name of a check type
+    ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
+    the corresponding check time or None if such a check time does not exist.
+
+    When the check type is "archives" or "data", this function probes two different paths to find
+    the check time, e.g.:
+
+      ~/.borgmatic/checks/1234567890/archives/9876543210
+      ~/.borgmatic/checks/1234567890/archives/all
+
+    ... and returns the modification time of the first file found (if any). The first path
+    represents a more specific archives check time (a check on a subset of archives), and the second
+    is a fallback to the last "all" archives check.
+
+    For other check types, this function reads from a single check time path, e.g.:
+
+      ~/.borgmatic/checks/1234567890/repository
+    '''
+    check_times = (
+        read_check_time(group[0])
+        for group in itertools.groupby(
+            (
+                make_check_time_path(location_config, borg_repository_id, check, archives_check_id),
+                make_check_time_path(location_config, borg_repository_id, check),
+            )
+        )
+    )
+
+    try:
+        return next(check_time for check_time in check_times if check_time)
+    except StopIteration:
+        return None
+
+
+def upgrade_check_times(location_config, borg_repository_id):
+    '''
+    Given a location configuration dict and a Borg repository ID, upgrade any corresponding check
+    times on disk from old-style paths to new-style paths.
+
+    Currently, the only upgrade performed is renaming an archive or data check path that looks like:
+
+      ~/.borgmatic/checks/1234567890/archives
+
+    to:
+
+      ~/.borgmatic/checks/1234567890/archives/all
+    '''
+    for check_type in ('archives', 'data'):
+        new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all')
+        old_path = os.path.dirname(new_path)
+        temporary_path = f'{old_path}.temp'
+
+        if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
+            return
+
+        logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
+
+        try:
+            os.rename(old_path, temporary_path)
+        except FileNotFoundError:
+            pass
+
+        os.mkdir(old_path)
+        os.rename(temporary_path, new_path)
+
+
 def check_archives(
     repository_path,
     location_config,
     storage_config,
     consistency_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     progress=None,
@@ -272,6 +386,7 @@ def check_archives(
                 storage_config,
                 local_borg_version,
                 argparse.Namespace(json=True),
+                global_arguments,
                 local_path,
                 remote_path,
             )
@@ -279,19 +394,29 @@ def check_archives(
     except (json.JSONDecodeError, KeyError):
         raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
 
+    upgrade_check_times(location_config, borg_repository_id)
+
+    check_last = consistency_config.get('check_last', None)
+    prefix = consistency_config.get('prefix')
+    configured_checks = parse_checks(consistency_config, only_checks)
+    lock_wait = None
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
+    archive_filter_flags = make_archive_filter_flags(
+        local_borg_version, storage_config, configured_checks, check_last, prefix
+    )
+    archives_check_id = make_archives_check_id(archive_filter_flags)
+
     checks = filter_checks_on_frequency(
         location_config,
         consistency_config,
         borg_repository_id,
-        parse_checks(consistency_config, only_checks),
+        configured_checks,
         force,
+        archives_check_id,
     )
-    check_last = consistency_config.get('check_last', None)
-    lock_wait = None
-    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
 
     if set(checks).intersection({'repository', 'archives', 'data'}):
-        lock_wait = storage_config.get('lock_wait', None)
+        lock_wait = storage_config.get('lock_wait')
 
         verbosity_flags = ()
         if logger.isEnabledFor(logging.INFO):
@@ -299,13 +424,12 @@ def check_archives(
         if logger.isEnabledFor(logging.DEBUG):
             verbosity_flags = ('--debug', '--show-rc')
 
-        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
-
         full_command = (
             (local_path, 'check')
             + (('--repair',) if repair else ())
-            + make_check_flags(local_borg_version, checks, check_last, prefix)
+            + make_check_flags(checks, archive_filter_flags)
             + (('--remote-path', remote_path) if remote_path else ())
+            + (('--log-json',) if global_arguments.log_json else ())
             + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
             + (('--progress',) if progress else ())
@@ -325,10 +449,18 @@ def check_archives(
             execute_command(full_command, extra_environment=borg_environment)
 
         for check in checks:
-            write_check_time(make_check_time_path(location_config, borg_repository_id, check))
+            write_check_time(
+                make_check_time_path(location_config, borg_repository_id, check, archives_check_id)
+            )
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(
-            storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path
+            storage_config,
+            local_borg_version,
+            global_arguments,
+            repository_path,
+            lock_wait,
+            local_path,
+            remote_path,
         )
         write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract'))

borgmatic/borg/compact.py

@@ -11,6 +11,7 @@ def compact_segments(
     repository_path,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -29,6 +30,7 @@ def compact_segments(
         (local_path, 'compact')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--progress',) if progress else ())
         + (('--cleanup-commits',) if cleanup_commits else ())

borgmatic/borg/create.py

@@ -314,7 +314,7 @@ def check_all_source_directories_exist(source_directories):
     missing_directories = [
         source_directory
         for source_directory in source_directories
-        if not os.path.exists(source_directory)
+        if not all([os.path.exists(directory) for directory in expand_directory(source_directory)])
     ]
     if missing_directories:
         raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
@@ -326,6 +326,7 @@ def create_archive(
     location_config,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -438,6 +439,7 @@ def create_archive(
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (
             ('--list', '--filter', list_filter_flags)
@@ -509,7 +511,9 @@ def create_archive(
         )
     elif output_log_level is None:
         return execute_command_and_capture_output(
-            create_command, working_directory=working_directory, extra_environment=borg_environment,
+            create_command,
+            working_directory=working_directory,
+            extra_environment=borg_environment,
         )
     else:
         execute_command(

borgmatic/borg/export_tar.py

@@ -15,6 +15,7 @@ def export_tar_archive(
     destination_path,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     tar_filter=None,
@@ -38,6 +39,7 @@ def export_tar_archive(
         (local_path, 'export-tar')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--list',) if list_files else ())
@@ -45,7 +47,11 @@ def export_tar_archive(
         + (('--dry-run',) if dry_run else ())
         + (('--tar-filter', tar_filter) if tar_filter else ())
         + (('--strip-components', str(strip_components)) if strip_components else ())
-        + flags.make_repository_archive_flags(repository_path, archive, local_borg_version,)
+        + flags.make_repository_archive_flags(
+            repository_path,
+            archive,
+            local_borg_version,
+        )
         + (destination_path,)
         + (tuple(paths) if paths else ())
     )

borgmatic/borg/extract.py

@@ -2,6 +2,7 @@ import logging
 import os
 import subprocess
 
+import borgmatic.config.validate
 from borgmatic.borg import environment, feature, flags, rlist
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
@@ -11,6 +12,7 @@ logger = logging.getLogger(__name__)
 def extract_last_archive_dry_run(
     storage_config,
     local_borg_version,
+    global_arguments,
     repository_path,
     lock_wait=None,
     local_path='borg',
@@ -20,8 +22,6 @@ def extract_last_archive_dry_run(
     Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
     dry-run.
     '''
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     verbosity_flags = ()
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
@@ -30,7 +30,13 @@ def extract_last_archive_dry_run(
 
     try:
         last_archive_name = rlist.resolve_archive_name(
-            repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path
+            repository_path,
+            'latest',
+            storage_config,
+            local_borg_version,
+            global_arguments,
+            local_path,
+            remote_path,
         )
     except ValueError:
         logger.warning('No archives found. Skipping extract consistency check.')
@@ -40,8 +46,9 @@ def extract_last_archive_dry_run(
     borg_environment = environment.make_environment(storage_config)
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
-        + remote_path_flags
-        + lock_wait_flags
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + verbosity_flags
         + list_flag
         + flags.make_repository_archive_flags(
@@ -62,6 +69,7 @@ def extract_archive(
     location_config,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     destination_path=None,
@@ -71,9 +79,9 @@ def extract_archive(
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    restore from the archive, the local Borg version string, location/storage configuration dicts,
-    optional local and remote Borg paths, and an optional destination path to extract to, extract
-    the archive into the current directory.
+    restore from the archive, the local Borg version string, an argparse.Namespace of global
+    arguments, location/storage configuration dicts, optional local and remote Borg paths, and an
+    optional destination path to extract to, extract the archive into the current directory.
 
     If extract to stdout is True, then start the extraction streaming to stdout, and return that
     extract process as an instance of subprocess.Popen.
@@ -101,6 +109,7 @@ def extract_archive(
         + (('--remote-path', remote_path) if remote_path else ())
         + numeric_ids_flags
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
@@ -108,7 +117,13 @@ def extract_archive(
         + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if progress else ())
         + (('--stdout',) if extract_to_stdout else ())
-        + flags.make_repository_archive_flags(repository, archive, local_borg_version,)
+        + flags.make_repository_archive_flags(
+            # Make the repository path absolute so the working directory changes below don't
+            # prevent Borg from finding the repo.
+            borgmatic.config.validate.normalize_repository_path(repository),
+            archive,
+            local_borg_version,
+        )
         + (tuple(paths) if paths else ())
     )
 

borgmatic/borg/feature.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
-from pkg_resources import parse_version
+from packaging.version import parse
 
 
 class Feature(Enum):
@@ -18,17 +18,17 @@ class Feature(Enum):
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
-    Feature.COMPACT: parse_version('1.2.0a2'),  # borg compact
-    Feature.ATIME: parse_version('1.2.0a7'),  # borg create --atime
-    Feature.NOFLAGS: parse_version('1.2.0a8'),  # borg create --noflags
-    Feature.NUMERIC_IDS: parse_version('1.2.0b3'),  # borg create/extract/mount --numeric-ids
-    Feature.UPLOAD_RATELIMIT: parse_version('1.2.0b3'),  # borg create --upload-ratelimit
-    Feature.SEPARATE_REPOSITORY_ARCHIVE: parse_version('2.0.0a2'),  # --repo with separate archive
-    Feature.RCREATE: parse_version('2.0.0a2'),  # borg rcreate
-    Feature.RLIST: parse_version('2.0.0a2'),  # borg rlist
-    Feature.RINFO: parse_version('2.0.0a2'),  # borg rinfo
-    Feature.MATCH_ARCHIVES: parse_version('2.0.0b3'),  # borg --match-archives
-    Feature.EXCLUDED_FILES_MINUS: parse_version('2.0.0b5'),  # --list --filter uses "-" for excludes
+    Feature.COMPACT: parse('1.2.0a2'),  # borg compact
+    Feature.ATIME: parse('1.2.0a7'),  # borg create --atime
+    Feature.NOFLAGS: parse('1.2.0a8'),  # borg create --noflags
+    Feature.NUMERIC_IDS: parse('1.2.0b3'),  # borg create/extract/mount --numeric-ids
+    Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'),  # borg create --upload-ratelimit
+    Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'),  # --repo with separate archive
+    Feature.RCREATE: parse('2.0.0a2'),  # borg rcreate
+    Feature.RLIST: parse('2.0.0a2'),  # borg rlist
+    Feature.RINFO: parse('2.0.0a2'),  # borg rinfo
+    Feature.MATCH_ARCHIVES: parse('2.0.0b3'),  # borg --match-archives
+    Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
 }
 
 
@@ -37,4 +37,4 @@ def available(feature, borg_version):
     Given a Borg Feature constant and a Borg version string, return whether that feature is
     available in that version of Borg.
     '''
-    return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse_version(borg_version)
+    return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse(borg_version)

borgmatic/borg/flags.py

@@ -1,4 +1,5 @@
 import itertools
+import re
 
 from borgmatic.borg import feature
 
@@ -56,3 +57,27 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
         if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
         else (f'{repository_path}::{archive}',)
     )
+
+
+def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
+    '''
+    Return match archives flags based on the given match archives value, if any. If it isn't set,
+    return match archives flags to match archives created with the given archive name format, if
+    any. This is done by replacing certain archive name format placeholders for ephemeral data (like
+    "{now}") with globs.
+    '''
+    if match_archives:
+        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
+            return ('--match-archives', match_archives)
+        else:
+            return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
+
+    if not archive_name_format:
+        return ()
+
+    derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
+
+    if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
+        return ('--match-archives', f'sh:{derived_match_archives}')
+    else:
+        return ('--glob-archives', f'{derived_match_archives}')

borgmatic/borg/info.py

@@ -12,13 +12,14 @@ def display_archives_info(
     storage_config,
     local_borg_version,
     info_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, and the
-    arguments to the info action, display summary information for Borg archives in the repository or
-    return JSON summary information.
+    Given a local or remote repository path, a storage config dict, the local Borg version, global
+    arguments as an argparse.Namespace, and the arguments to the info action, display summary
+    information for Borg archives in the repository or return JSON summary information.
     '''
     borgmatic.logger.add_custom_log_levels()
     lock_wait = storage_config.get('lock_wait', None)
@@ -36,6 +37,7 @@ def display_archives_info(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
         + (
             (
@@ -44,22 +46,26 @@ def display_archives_info(
                 else flags.make_flags('glob-archives', f'{info_arguments.prefix}*')
             )
             if info_arguments.prefix
-            else ()
+            else (
+                flags.make_match_archives_flags(
+                    info_arguments.match_archives
+                    or info_arguments.archive
+                    or storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
         )
         + flags.make_flags_from_arguments(
-            info_arguments, excludes=('repository', 'archive', 'prefix')
+            info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
         )
         + flags.make_repository_flags(repository_path, local_borg_version)
-        + (
-            flags.make_flags('match-archives', info_arguments.archive)
-            if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-            else flags.make_flags('glob-archives', info_arguments.archive)
-        )
     )
 
     if info_arguments.json:
         return execute_command_and_capture_output(
-            full_command, extra_environment=environment.make_environment(storage_config),
+            full_command,
+            extra_environment=environment.make_environment(storage_config),
         )
     else:
         execute_command(

borgmatic/borg/list.py

@@ -25,6 +25,7 @@ def make_list_command(
     storage_config,
     local_borg_version,
     list_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
@@ -48,6 +49,7 @@ def make_list_command(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (
@@ -90,14 +92,16 @@ def capture_archive_listing(
     archive,
     storage_config,
     local_borg_version,
+    global_arguments,
     list_path=None,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, an archive name, a storage config dict, the local Borg
-    version, the archive path in which to list files, and local and remote Borg paths, capture the
-    output of listing that archive and return it as a list of file paths.
+    version, global arguments as an argparse.Namespace, the archive path in which to list files, and
+    local and remote Borg paths, capture the output of listing that archive and return it as a list
+    of file paths.
     '''
     borg_environment = environment.make_environment(storage_config)
 
@@ -115,6 +119,7 @@ def capture_archive_listing(
                     json=None,
                     format='{path}{NL}',  # noqa: FS003
                 ),
+                global_arguments,
                 local_path,
                 remote_path,
             ),
@@ -130,15 +135,17 @@ def list_archive(
     storage_config,
     local_borg_version,
     list_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the list action, and local and remote Borg paths, display the output of listing
-    the files of a Borg archive (or return JSON output). If list_arguments.find_paths are given,
-    list the files by searching across multiple archives. If neither find_paths nor archive name
-    are given, instead list the archives in the given repository.
+    Given a local or remote repository path, a storage config dict, the local Borg version, global
+    arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace,
+    and local and remote Borg paths, display the output of listing the files of a Borg archive (or
+    return JSON output). If list_arguments.find_paths are given, list the files by searching across
+    multiple archives. If neither find_paths nor archive name are given, instead list the archives
+    in the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
 
@@ -164,6 +171,7 @@ def list_archive(
             storage_config,
             local_borg_version,
             rlist_arguments,
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -205,6 +213,7 @@ def list_archive(
                     storage_config,
                     local_borg_version,
                     rlist_arguments,
+                    global_arguments,
                     local_path,
                     remote_path,
                 ),
@@ -233,6 +242,7 @@ def list_archive(
             storage_config,
             local_borg_version,
             archive_arguments,
+            global_arguments,
             local_path,
             remote_path,
         ) + make_find_paths(list_arguments.find_paths)

borgmatic/borg/mount.py

@@ -15,14 +15,15 @@ def mount_archive(
     options,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, an optional archive name, a filesystem mount point,
     zero or more paths to mount from the archive, extra Borg mount options, a storage configuration
-    dict, the local Borg version, and optional local and remote Borg paths, mount the archive onto
-    the mount point.
+    dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional
+    local and remote Borg paths, mount the archive onto the mount point.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
@@ -31,6 +32,7 @@ def mount_archive(
         (local_path, 'mount')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())

borgmatic/borg/prune.py

@@ -7,10 +7,10 @@ from borgmatic.execute import execute_command
 logger = logging.getLogger(__name__)
 
 
-def make_prune_flags(retention_config, local_borg_version):
+def make_prune_flags(storage_config, retention_config, local_borg_version):
     '''
-    Given a retention config dict mapping from option name to value, tranform it into an iterable of
-    command-line name-value flag pairs.
+    Given a retention config dict mapping from option name to value, transform it into an sequence of
+    command-line flags.
 
     For example, given a retention config of:
 
@@ -24,18 +24,28 @@ def make_prune_flags(retention_config, local_borg_version):
         )
     '''
     config = retention_config.copy()
-    prefix = config.pop('prefix', '{hostname}-')  # noqa: FS003
+    prefix = config.pop('prefix', None)
 
-    if prefix:
-        if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
-            config['match_archives'] = f'sh:{prefix}*'
-        else:
-            config['glob_archives'] = f'{prefix}*'
-
-    return (
+    flag_pairs = (
         ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
     )
 
+    return tuple(element for pair in flag_pairs for element in pair) + (
+        (
+            ('--match-archives', f'sh:{prefix}*')
+            if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
+            else ('--glob-archives', f'{prefix}*')
+        )
+        if prefix
+        else (
+            flags.make_match_archives_flags(
+                storage_config.get('match_archives'),
+                storage_config.get('archive_name_format'),
+                local_borg_version,
+            )
+        )
+    )
+
 
 def prune_archives(
     dry_run,
@@ -43,6 +53,7 @@ def prune_archives(
     storage_config,
     retention_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     stats=False,
@@ -60,13 +71,10 @@ def prune_archives(
 
     full_command = (
         (local_path, 'prune')
-        + tuple(
-            element
-            for pair in make_prune_flags(retention_config, local_borg_version)
-            for element in pair
-        )
+        + make_prune_flags(storage_config, retention_config, local_borg_version)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--stats',) if stats and not dry_run else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())

borgmatic/borg/rcreate.py

@@ -16,6 +16,7 @@ def create_repository(
     repository_path,
     storage_config,
     local_borg_version,
+    global_arguments,
     encryption_mode,
     source_repository=None,
     copy_crypt_key=False,
@@ -37,6 +38,7 @@ def create_repository(
             storage_config,
             local_borg_version,
             argparse.Namespace(json=True),
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -46,6 +48,7 @@ def create_repository(
         if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
             raise
 
+    lock_wait = storage_config.get('lock_wait')
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '')
 
     rcreate_command = (
@@ -63,6 +66,8 @@ def create_repository(
         + (('--make-parent-dirs',) if make_parent_dirs else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)

borgmatic/borg/rinfo.py

@@ -12,13 +12,14 @@ def display_repository_info(
     storage_config,
     local_borg_version,
     rinfo_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, and the
-    arguments to the rinfo action, display summary information for the Borg repository or return
-    JSON summary information.
+    Given a local or remote repository path, a storage config dict, the local Borg version, the
+    arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary
+    information for the Borg repository or return JSON summary information.
     '''
     borgmatic.logger.add_custom_log_levels()
     lock_wait = storage_config.get('lock_wait', None)
@@ -41,6 +42,7 @@ def display_repository_info(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
         + (('--json',) if rinfo_arguments.json else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
@@ -50,7 +52,8 @@ def display_repository_info(
 
     if rinfo_arguments.json:
         return execute_command_and_capture_output(
-            full_command, extra_environment=extra_environment,
+            full_command,
+            extra_environment=extra_environment,
         )
     else:
         execute_command(

borgmatic/borg/rlist.py

@@ -12,35 +12,37 @@ def resolve_archive_name(
     archive,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, an archive name, a storage config dict, a local Borg
-    path, and a remote Borg path, simply return the archive name. But if the archive name is
-    "latest", then instead introspect the repository for the latest archive and return its name.
+    Given a local or remote repository path, an archive name, a storage config dict, the local Borg
+    version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
+    return the archive name. But if the archive name is "latest", then instead introspect the
+    repository for the latest archive and return its name.
 
     Raise ValueError if "latest" is given but there are no archives in the repository.
     '''
     if archive != 'latest':
         return archive
 
-    lock_wait = storage_config.get('lock_wait', None)
-
     full_command = (
         (
             local_path,
             'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list',
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('lock-wait', lock_wait)
+        + flags.make_flags('log-json', global_arguments.log_json)
+        + flags.make_flags('lock-wait', storage_config.get('lock_wait'))
         + flags.make_flags('last', 1)
         + ('--short',)
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     output = execute_command_and_capture_output(
-        full_command, extra_environment=environment.make_environment(storage_config),
+        full_command,
+        extra_environment=environment.make_environment(storage_config),
     )
     try:
         latest_archive = output.strip().splitlines()[-1]
@@ -52,7 +54,7 @@ def resolve_archive_name(
     return latest_archive
 
 
-MAKE_FLAGS_EXCLUDES = ('repository', 'prefix')
+MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
 
 
 def make_rlist_command(
@@ -60,16 +62,15 @@ def make_rlist_command(
     storage_config,
     local_borg_version,
     rlist_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the rlist action, and local and remote Borg paths, return a command as a tuple to
-    list archives with a repository.
+    arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and
+    remote Borg paths, return a command as a tuple to list archives with a repository.
     '''
-    lock_wait = storage_config.get('lock_wait', None)
-
     return (
         (
             local_path,
@@ -86,7 +87,8 @@ def make_rlist_command(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('lock-wait', lock_wait)
+        + flags.make_flags('log-json', global_arguments.log_json)
+        + flags.make_flags('lock-wait', storage_config.get('lock_wait'))
         + (
             (
                 flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*')
@@ -94,7 +96,13 @@ def make_rlist_command(
                 else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*')
             )
             if rlist_arguments.prefix
-            else ()
+            else (
+                flags.make_match_archives_flags(
+                    rlist_arguments.match_archives or storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
         )
         + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + flags.make_repository_flags(repository_path, local_borg_version)
@@ -106,13 +114,15 @@ def list_repository(
     storage_config,
     local_borg_version,
     rlist_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the list action, and local and remote Borg paths, display the output of listing
-    Borg archives in the given repository (or return JSON output).
+    arguments to the list action, global arguments as an argparse.Namespace instance, and local and
+    remote Borg paths, display the output of listing Borg archives in the given repository (or
+    return JSON output).
     '''
     borgmatic.logger.add_custom_log_levels()
     borg_environment = environment.make_environment(storage_config)
@@ -122,6 +132,7 @@ def list_repository(
         storage_config,
         local_borg_version,
         rlist_arguments,
+        global_arguments,
         local_path,
         remote_path,
     )

borgmatic/borg/transfer.py

@@ -13,12 +13,14 @@ def transfer_archives(
     storage_config,
     local_borg_version,
     transfer_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg
-    version, and the arguments to the transfer action, transfer archives to the given repository.
+    version, the arguments to the transfer action, and global arguments as an argparse.Namespace
+    instance, transfer archives to the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
 
@@ -27,16 +29,22 @@ def transfer_archives(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', storage_config.get('lock_wait', None))
-        + (('--progress',) if transfer_arguments.progress else ())
         + (
-            flags.make_flags(
-                'match-archives', transfer_arguments.match_archives or transfer_arguments.archive
+            flags.make_flags_from_arguments(
+                transfer_arguments,
+                excludes=('repository', 'source_repository', 'archive', 'match_archives'),
+            )
+            or (
+                flags.make_match_archives_flags(
+                    transfer_arguments.match_archives
+                    or transfer_arguments.archive
+                    or storage_config.get('match_archives'),
+                    storage_config.get('archive_name_format'),
+                    local_borg_version,
+                )
             )
-        )
-        + flags.make_flags_from_arguments(
-            transfer_arguments,
-            excludes=('repository', 'source_repository', 'archive', 'match_archives'),
         )
         + flags.make_repository_flags(repository_path, local_borg_version)
         + flags.make_flags('other-repo', transfer_arguments.source_repository)

borgmatic/borg/version.py

@@ -19,7 +19,8 @@ def local_borg_version(storage_config, local_path='borg'):
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
     )
     output = execute_command_and_capture_output(
-        full_command, extra_environment=environment.make_environment(storage_config),
+        full_command,
+        extra_environment=environment.make_environment(storage_config),
     )
 
     try:

borgmatic/commands/arguments.py

@@ -180,9 +180,18 @@ def make_parsers():
     global_group.add_argument(
         '--log-file',
         type=str,
-        default=None,
         help='Write log messages to this file instead of syslog',
     )
+    global_group.add_argument(
+        '--log-file-format',
+        type=str,
+        help='Log format string used for log messages written to the log file',
+    )
+    global_group.add_argument(
+        '--log-json',
+        action='store_true',
+        help='Write log messages and console output as one JSON object per log line instead of formatted text',
+    )
     global_group.add_argument(
         '--override',
         metavar='SECTION.OPTION=VALUE',
@@ -203,6 +212,12 @@ def make_parsers():
         action='store_true',
         help='Show bash completion script and exit',
     )
+    global_group.add_argument(
+        '--fish-completion',
+        default=False,
+        action='store_true',
+        help='Show fish completion script and exit',
+    )
     global_group.add_argument(
         '--version',
         dest='version',
@@ -256,10 +271,13 @@ def make_parsers():
         help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)',
     )
     rcreate_group.add_argument(
-        '--append-only', action='store_true', help='Create an append-only repository',
+        '--append-only',
+        action='store_true',
+        help='Create an append-only repository',
     )
     rcreate_group.add_argument(
-        '--storage-quota', help='Create a repository with a fixed storage quota',
+        '--storage-quota',
+        help='Create a repository with a fixed storage quota',
     )
     rcreate_group.add_argument(
         '--make-parent-dirs',
@@ -293,7 +311,7 @@ def make_parsers():
     )
     transfer_group.add_argument(
         '--upgrader',
-        help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion',
+        help='Upgrader type used to convert the transferred data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion',
     )
     transfer_group.add_argument(
         '--progress',
@@ -627,6 +645,13 @@ def make_parsers():
         dest='databases',
         help="Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic's configuration",
     )
+    restore_group.add_argument(
+        '--schema',
+        metavar='NAME',
+        nargs='+',
+        dest='schemas',
+        help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
+    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
@@ -640,7 +665,8 @@ def make_parsers():
     )
     rlist_group = rlist_parser.add_argument_group('rlist arguments')
     rlist_group.add_argument(
-        '--repository', help='Path of repository to list, defaults to the configured repositories',
+        '--repository',
+        help='Path of repository to list, defaults to the configured repositories',
     )
     rlist_group.add_argument(
         '--short', default=False, action='store_true', help='Output only archive names'
@@ -650,7 +676,7 @@ def make_parsers():
         '--json', default=False, action='store_true', help='Output results as JSON'
     )
     rlist_group.add_argument(
-        '-P', '--prefix', help='Only list archive names starting with this prefix'
+        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
     )
     rlist_group.add_argument(
         '-a',
@@ -705,7 +731,7 @@ def make_parsers():
         '--json', default=False, action='store_true', help='Output results as JSON'
     )
     list_group.add_argument(
-        '-P', '--prefix', help='Only list archive names starting with this prefix'
+        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
     )
     list_group.add_argument(
         '-a',
@@ -777,7 +803,9 @@ def make_parsers():
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
     info_group.add_argument(
-        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+        '-P',
+        '--prefix',
+        help='Deprecated. Only show info for archive names starting with this prefix',
     )
     info_group.add_argument(
         '-a',
@@ -861,6 +889,10 @@ def parse_arguments(*unparsed_arguments):
         raise ValueError(
             'With the create action, only one of --list (--files) and --progress flags can be used.'
         )
+    if 'create' in arguments and arguments['create'].list_files and arguments['create'].json:
+        raise ValueError(
+            'With the create action, only one of --list (--files) and --json flags can be used.'
+        )
 
     if (
         ('list' in arguments and 'rinfo' in arguments and arguments['list'].json)
@@ -875,7 +907,17 @@ def parse_arguments(*unparsed_arguments):
         and arguments['transfer'].match_archives
     ):
         raise ValueError(
-            'With the transfer action, only one of --archive and --glob-archives flags can be used.'
+            'With the transfer action, only one of --archive and --match-archives flags can be used.'
+        )
+
+    if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives):
+        raise ValueError(
+            'With the list action, only one of --prefix or --match-archives flags can be used.'
+        )
+
+    if 'rlist' in arguments and (arguments['rlist'].prefix and arguments['rlist'].match_archives):
+        raise ValueError(
+            'With the rlist action, only one of --prefix or --match-archives flags can be used.'
         )
 
     if 'info' in arguments and (

borgmatic/commands/borgmatic.py

@@ -8,7 +8,11 @@ from queue import Queue
 from subprocess import CalledProcessError
 
 import colorama
-import pkg_resources
+
+try:
+    import importlib_metadata
+except ModuleNotFoundError:  # pragma: nocover
+    import importlib.metadata as importlib_metadata
 
 import borgmatic.actions.borg
 import borgmatic.actions.break_lock
@@ -103,14 +107,20 @@ def run_configuration(config_filename, config, arguments):
     if not encountered_error:
         repo_queue = Queue()
         for repo in location['repositories']:
-            repo_queue.put((repo, 0),)
+            repo_queue.put(
+                (repo, 0),
+            )
 
         while not repo_queue.empty():
             repository, retry_num = repo_queue.get()
-            logger.debug(f'{repository["path"]}: Running actions for repository')
+            logger.debug(
+                f'{repository.get("label", repository["path"])}: Running actions for repository'
+            )
             timeout = retry_num * retry_wait
             if timeout:
-                logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
+                logger.warning(
+                    f'{repository.get("label", repository["path"])}: Sleeping {timeout}s before next retry'
+                )
                 time.sleep(timeout)
             try:
                 yield from run_actions(
@@ -128,17 +138,19 @@ def run_configuration(config_filename, config, arguments):
                 )
             except (OSError, CalledProcessError, ValueError) as error:
                 if retry_num < retries:
-                    repo_queue.put((repository, retry_num + 1),)
+                    repo_queue.put(
+                        (repository, retry_num + 1),
+                    )
                     tuple(  # Consume the generator so as to trigger logging.
                         log_error_records(
-                            f'{repository["path"]}: Error running actions for repository',
+                            f'{repository.get("label", repository["path"])}: Error running actions for repository',
                             error,
                             levelno=logging.WARNING,
                             log_command_error_output=True,
                         )
                     )
                     logger.warning(
-                        f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}'
+                        f'{repository.get("label", repository["path"])}: Retrying... attempt {retry_num + 1}/{retries}'
                     )
                     continue
 
@@ -146,7 +158,8 @@ def run_configuration(config_filename, config, arguments):
                     return
 
                 yield from log_error_records(
-                    f'{repository["path"]}: Error running actions for repository', error
+                    f'{repository.get("label", repository["path"])}: Error running actions for repository',
+                    error,
                 )
                 encountered_error = error
                 error_repository = repository['path']
@@ -267,6 +280,7 @@ def run_actions(
         'repository': repository_path,
         # Deprecated: For backwards compatibility with borgmatic < 1.6.0.
         'repositories': ','.join([repo['path'] for repo in location['repositories']]),
+        'log_file': global_arguments.log_file if global_arguments.log_file else '',
     }
 
     command.execute_hook(
@@ -278,7 +292,7 @@ def run_actions(
         **hook_context,
     )
 
-    for (action_name, action_arguments) in arguments.items():
+    for action_name, action_arguments in arguments.items():
         if action_name == 'rcreate':
             borgmatic.actions.rcreate.run_rcreate(
                 repository,
@@ -389,7 +403,8 @@ def run_actions(
                 repository,
                 storage,
                 local_borg_version,
-                arguments['mount'],
+                action_arguments,
+                global_arguments,
                 local_path,
                 remote_path,
             )
@@ -407,32 +422,63 @@ def run_actions(
             )
         elif action_name == 'rlist':
             yield from borgmatic.actions.rlist.run_rlist(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'list':
             yield from borgmatic.actions.list.run_list(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'rinfo':
             yield from borgmatic.actions.rinfo.run_rinfo(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'info':
             yield from borgmatic.actions.info.run_info(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'break-lock':
             borgmatic.actions.break_lock.run_break_lock(
                 repository,
                 storage,
                 local_borg_version,
-                arguments['break-lock'],
+                action_arguments,
+                global_arguments,
                 local_path,
                 remote_path,
             )
         elif action_name == 'borg':
             borgmatic.actions.borg.run_borg(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
 
     command.execute_hook(
@@ -625,7 +671,8 @@ def collect_configuration_run_summary_logs(configs, arguments):
         logger.info(f"Unmounting mount point {arguments['umount'].mount_point}")
         try:
             borg_umount.unmount_archive(
-                mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs),
+                mount_point=arguments['umount'].mount_point,
+                local_path=get_local_path(configs),
             )
         except (CalledProcessError, OSError) as error:
             yield from log_error_records('Error unmounting mount point', error)
@@ -675,11 +722,14 @@ def main():  # pragma: no cover
 
     global_arguments = arguments['global']
     if global_arguments.version:
-        print(pkg_resources.require('borgmatic')[0].version)
+        print(importlib_metadata.version('borgmatic'))
         sys.exit(0)
     if global_arguments.bash_completion:
         print(borgmatic.commands.completion.bash_completion())
         sys.exit(0)
+    if global_arguments.fish_completion:
+        print(borgmatic.commands.completion.fish_completion())
+        sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
     configs, parse_logs = load_configurations(
@@ -700,6 +750,7 @@ def main():  # pragma: no cover
             verbosity_to_log_level(global_arguments.log_file_verbosity),
             verbosity_to_log_level(global_arguments.monitoring_verbosity),
             global_arguments.log_file,
+            global_arguments.log_file_format,
         )
     except (FileNotFoundError, PermissionError) as error:
         configure_logging(logging.CRITICAL)

borgmatic/commands/completion.py

@@ -1,12 +1,18 @@
+import shlex
+from argparse import Action
+from textwrap import dedent
+
 from borgmatic.commands import arguments
 
-UPGRADE_MESSAGE = '''
-Your bash completions script is from a different version of borgmatic than is
+
+def upgrade_message(language: str, upgrade_command: str, completion_file: str):
+    return f'''
+Your {language} completions script is from a different version of borgmatic than is
 currently installed. Please upgrade your script so your completions match the
 command-line flags in your installed borgmatic! Try this to upgrade:
 
-    sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"
-    source $BASH_SOURCE
+    {upgrade_command}
+    source {completion_file}
 '''
 
 
@@ -34,7 +40,11 @@ def bash_completion():
             '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
             '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
             '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
-            f'        then cat << EOF\n{UPGRADE_MESSAGE}\nEOF',
+            f'''        then cat << EOF\n{upgrade_message(
+                    'bash',
+                    'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
+                    '$BASH_SOURCE',
+                )}\nEOF''',
             '    fi',
             '}',
             'complete_borgmatic() {',
@@ -55,3 +65,172 @@ def bash_completion():
             '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
         )
     )
+
+
+# fish section
+
+
+def has_file_options(action: Action):
+    '''
+    Given an argparse.Action instance, return True if it takes a file argument.
+    '''
+    return action.metavar in (
+        'FILENAME',
+        'PATH',
+    ) or action.dest in ('config_paths',)
+
+
+def has_choice_options(action: Action):
+    '''
+    Given an argparse.Action instance, return True if it takes one of a predefined set of arguments.
+    '''
+    return action.choices is not None
+
+
+def has_unknown_required_param_options(action: Action):
+    '''
+    A catch-all for options that take a required parameter, but we don't know what the parameter is.
+    This should be used last. These are actions that take something like a glob, a list of numbers, or a string.
+
+    Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid.
+    '''
+    return (
+        action.required is True
+        or action.nargs
+        in (
+            '+',
+            '*',
+        )
+        or action.metavar in ('PATTERN', 'KEYS', 'N')
+        or (action.type is not None and action.default is None)
+    )
+
+
+def has_exact_options(action: Action):
+    return (
+        has_file_options(action)
+        or has_choice_options(action)
+        or has_unknown_required_param_options(action)
+    )
+
+
+def exact_options_completion(action: Action):
+    '''
+    Given an argparse.Action instance, return a completion invocation that forces file completions, options completion,
+    or just that some value follow the action, if the action takes such an argument and was the last action on the
+    command line prior to the cursor.
+
+    Otherwise, return an empty string.
+    '''
+
+    if not has_exact_options(action):
+        return ''
+
+    args = ' '.join(action.option_strings)
+
+    if has_file_options(action):
+        return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_current_arg {args}"'''
+
+    if has_choice_options(action):
+        return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_current_arg {args}"'''
+
+    if has_unknown_required_param_options(action):
+        return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"'''
+
+    raise ValueError(
+        f'Unexpected action: {action} passes has_exact_options but has no choices produced'
+    )
+
+
+def dedent_strip_as_tuple(string: str):
+    '''
+    Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string.
+    Makes it easier to write multiline strings for completions when you join them with a tuple.
+    '''
+    return (dedent(string).strip('\n'),)
+
+
+def fish_completion():
+    '''
+    Return a fish completion script for the borgmatic command. Produce this by introspecting
+    borgmatic's command-line argument parsers.
+    '''
+    top_level_parser, subparsers = arguments.make_parsers()
+
+    all_subparsers = ' '.join(action for action in subparsers.choices.keys())
+
+    exact_option_args = tuple(
+        ' '.join(action.option_strings)
+        for subparser in subparsers.choices.values()
+        for action in subparser._actions
+        if has_exact_options(action)
+    ) + tuple(
+        ' '.join(action.option_strings)
+        for action in top_level_parser._actions
+        if len(action.option_strings) > 0
+        if has_exact_options(action)
+    )
+
+    # Avert your eyes.
+    return '\n'.join(
+        dedent_strip_as_tuple(
+            f'''
+            function __borgmatic_check_version
+                set -fx this_filename (status current-filename)
+                fish -c '
+                    if test -f "$this_filename"
+                        set this_script (cat $this_filename 2> /dev/null)
+                        set installed_script (borgmatic --fish-completion 2> /dev/null)
+                        if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]
+                            echo "{upgrade_message(
+                            'fish',
+                            'borgmatic --fish-completion | sudo tee $this_filename',
+                            '$this_filename',
+                        )}"
+                        end
+                    end
+                ' &
+            end
+            __borgmatic_check_version
+
+            function __borgmatic_current_arg --description 'Check if any of the given arguments are the last on the command line before the cursor'
+                set -l all_args (commandline -poc)
+                # premature optimization to avoid iterating all args if there aren't enough
+                # to have a last arg beyond borgmatic
+                if [ (count $all_args) -lt 2 ]
+                    return 1
+                end
+                for arg in $argv
+                    if [ "$arg" = "$all_args[-1]" ]
+                        return 0
+                    end
+                end
+                return 1
+            end
+
+            set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"
+            set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
+            '''
+        )
+        + ('\n# subparser completions',)
+        + tuple(
+            f'''complete -c borgmatic -f -n "$subparser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}'''
+            for action_name, subparser in subparsers.choices.items()
+        )
+        + ('\n# global flags',)
+        + tuple(
+            # -n is checked in order, so put faster / more likely to be true checks first
+            f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
+            for action in top_level_parser._actions
+            # ignore the noargs action, as this is an impossible completion for fish
+            if len(action.option_strings) > 0
+            if 'Deprecated' not in action.help
+        )
+        + ('\n# subparser flags',)
+        + tuple(
+            f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
+            for action_name, subparser in subparsers.choices.items()
+            for action in subparser._actions
+            if 'Deprecated' not in action.help
+        )
+    )

borgmatic/commands/validate_config.py

@@ -2,6 +2,7 @@ import logging
 import sys
 from argparse import ArgumentParser
 
+import borgmatic.config.generate
 from borgmatic.config import collect, validate
 
 logger = logging.getLogger(__name__)
@@ -23,16 +24,22 @@ def parse_arguments(*arguments):
         default=config_paths,
         help=f'Configuration filenames or directories, defaults to: {config_paths}',
     )
+    parser.add_argument(
+        '-s',
+        '--show',
+        action='store_true',
+        help='Show the validated configuration after all include merging has occurred',
+    )
 
     return parser.parse_args(arguments)
 
 
 def main():  # pragma: no cover
-    args = parse_arguments(*sys.argv[1:])
+    arguments = parse_arguments(*sys.argv[1:])
 
     logging.basicConfig(level=logging.INFO, format='%(message)s')
 
-    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths))
     if len(config_filenames) == 0:
         logger.critical('No files to validate found')
         sys.exit(1)
@@ -40,13 +47,22 @@ def main():  # pragma: no cover
     found_issues = False
     for config_filename in config_filenames:
         try:
-            validate.parse_configuration(config_filename, validate.schema_filename())
+            config, parse_logs = validate.parse_configuration(
+                config_filename, validate.schema_filename()
+            )
         except (ValueError, OSError, validate.Validation_error) as error:
             logging.critical(f'{config_filename}: Error parsing configuration file')
             logging.critical(error)
             found_issues = True
+        else:
+            for log in parse_logs:
+                logger.handle(log)
+
+            if arguments.show:
+                print('---')
+                print(borgmatic.config.generate.render_configuration(config))
 
     if found_issues:
         sys.exit(1)
-    else:
-        logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")
+
+    logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")

borgmatic/config/convert.py

@@ -43,7 +43,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         ]
     )
 
-    # Split space-seperated values into actual lists, make "repository" into a list, and merge in
+    # Split space-separated values into actual lists, make "repository" into a list, and merge in
     # excludes.
     location = destination_config['location']
     location['source_directories'] = source_config.location['source_directories'].split(' ')

borgmatic/config/generate.py

@@ -260,7 +260,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
             )
             continue
 
-        # This is some sort of scalar. Simply set it into the destination.
+        # This is some sort of scalar. Set it into the destination.
         destination_config[field_name] = source_config[field_name]
 
     return destination_config

borgmatic/config/load.py

@@ -38,6 +38,37 @@ def include_configuration(loader, filename_node, include_directory):
     return load_configuration(include_filename)
 
 
+def raise_retain_node_error(loader, node):
+    '''
+    Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!retain" usage.
+
+    Raise ValueError if a mapping or sequence node is given, as that indicates that "!retain" was
+    used in a configuration file without a merge. In configuration files with a merge, mapping and
+    sequence nodes with "!retain" tags are handled by deep_merge_nodes() below.
+
+    Also raise ValueError if a scalar node is given, as "!retain" is not supported on scalar nodes.
+    '''
+    if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)):
+        raise ValueError(
+            'The !retain tag may only be used within a configuration file containing a merged !include tag.'
+        )
+
+    raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
+
+
+def raise_omit_node_error(loader, node):
+    '''
+    Given a ruamel.yaml.loader.Loader and a YAML node, raise an error about "!omit" usage.
+
+    Raise ValueError unconditionally, as an "!omit" node here indicates it was used in a
+    configuration file without a merge. In configuration files with a merge, nodes with "!omit"
+    tags are handled by deep_merge_nodes() below.
+    '''
+    raise ValueError(
+        'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.'
+    )
+
+
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
@@ -50,6 +81,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
             '!include',
             functools.partial(include_configuration, include_directory=include_directory),
         )
+        self.add_constructor('!retain', raise_retain_node_error)
+        self.add_constructor('!omit', raise_omit_node_error)
 
     def flatten_mapping(self, node):
         '''
@@ -88,6 +121,7 @@ def load_configuration(filename):
     Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
     if there are too many recursive includes.
     '''
+
     # Use an embedded derived class for the include constructor so as to capture the filename
     # value. (functools.partial doesn't work for this use case because yaml.Constructor has to be
     # an actual class.)
@@ -115,6 +149,16 @@ def load_configuration(filename):
         return config
 
 
+def filter_omitted_nodes(nodes):
+    '''
+    Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a
+    value matching such nodes.
+    '''
+    omitted_values = tuple(node.value for node in nodes if node.tag == '!omit')
+
+    return [node for node in nodes if node.value not in omitted_values]
+
+
 DELETED_NODE = object()
 
 
@@ -176,6 +220,8 @@ def deep_merge_nodes(nodes):
             ),
         ]
 
+    If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged.
+
     The purpose of deep merging like this is to support, for instance, merging one borgmatic
     configuration file into another for reuse, such that a configuration section ("retention",
     etc.) does not completely replace the corresponding section in a merged file.
@@ -198,32 +244,42 @@ def deep_merge_nodes(nodes):
 
                 # If we're dealing with MappingNodes, recurse and merge its values as well.
                 if isinstance(b_value, ruamel.yaml.nodes.MappingNode):
-                    replaced_nodes[(b_key, b_value)] = (
-                        b_key,
-                        ruamel.yaml.nodes.MappingNode(
-                            tag=b_value.tag,
-                            value=deep_merge_nodes(a_value.value + b_value.value),
-                            start_mark=b_value.start_mark,
-                            end_mark=b_value.end_mark,
-                            flow_style=b_value.flow_style,
-                            comment=b_value.comment,
-                            anchor=b_value.anchor,
-                        ),
-                    )
+                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+                    # downstream schema validation doesn't break on our application-specific tag.
+                    if b_value.tag == '!retain':
+                        b_value.tag = 'tag:yaml.org,2002:map'
+                    else:
+                        replaced_nodes[(b_key, b_value)] = (
+                            b_key,
+                            ruamel.yaml.nodes.MappingNode(
+                                tag=b_value.tag,
+                                value=deep_merge_nodes(a_value.value + b_value.value),
+                                start_mark=b_value.start_mark,
+                                end_mark=b_value.end_mark,
+                                flow_style=b_value.flow_style,
+                                comment=b_value.comment,
+                                anchor=b_value.anchor,
+                            ),
+                        )
                 # If we're dealing with SequenceNodes, merge by appending one sequence to the other.
                 elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode):
-                    replaced_nodes[(b_key, b_value)] = (
-                        b_key,
-                        ruamel.yaml.nodes.SequenceNode(
-                            tag=b_value.tag,
-                            value=a_value.value + b_value.value,
-                            start_mark=b_value.start_mark,
-                            end_mark=b_value.end_mark,
-                            flow_style=b_value.flow_style,
-                            comment=b_value.comment,
-                            anchor=b_value.anchor,
-                        ),
-                    )
+                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+                    # downstream schema validation doesn't break on our application-specific tag.
+                    if b_value.tag == '!retain':
+                        b_value.tag = 'tag:yaml.org,2002:seq'
+                    else:
+                        replaced_nodes[(b_key, b_value)] = (
+                            b_key,
+                            ruamel.yaml.nodes.SequenceNode(
+                                tag=b_value.tag,
+                                value=filter_omitted_nodes(a_value.value + b_value.value),
+                                start_mark=b_value.start_mark,
+                                end_mark=b_value.end_mark,
+                                flow_style=b_value.flow_style,
+                                comment=b_value.comment,
+                                anchor=b_value.anchor,
+                            ),
+                        )
 
     return [
         replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE

borgmatic/config/normalize.py

@@ -81,7 +81,10 @@ def normalize(config_filename, config):
                         repository_path.partition('file://')[-1]
                     )
                     config['location']['repositories'].append(
-                        dict(repository_dict, path=updated_repository_path,)
+                        dict(
+                            repository_dict,
+                            path=updated_repository_path,
+                        )
                     )
                 elif repository_path.startswith('ssh://'):
                     config['location']['repositories'].append(repository_dict)
@@ -97,7 +100,10 @@ def normalize(config_filename, config):
                         )
                     )
                     config['location']['repositories'].append(
-                        dict(repository_dict, path=rewritten_repository_path,)
+                        dict(
+                            repository_dict,
+                            path=rewritten_repository_path,
+                        )
                     )
             else:
                 config['location']['repositories'].append(repository_dict)

borgmatic/config/override.py

@@ -57,7 +57,12 @@ def parse_overrides(raw_overrides):
     for raw_override in raw_overrides:
         try:
             raw_keys, value = raw_override.split('=', 1)
-            parsed_overrides.append((tuple(raw_keys.split('.')), convert_value_type(value),))
+            parsed_overrides.append(
+                (
+                    tuple(raw_keys.split('.')),
+                    convert_value_type(value),
+                )
+            )
         except ValueError:
             raise ValueError(
                 f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE"
@@ -75,5 +80,5 @@ def apply_overrides(config, raw_overrides):
     '''
     overrides = parse_overrides(raw_overrides)
 
-    for (keys, value) in overrides:
+    for keys, value in overrides:
         set_values(config, keys, value)

borgmatic/config/schema.yaml

@@ -6,7 +6,7 @@ properties:
     constants:
         type: object
         description: |
-            Constants to use in the configuration file. All occurences of the
+            Constants to use in the configuration file. All occurrences of the
             constant name within culy braces will be replaced with the value.
             For example, if you have a constant named "hostname" with the value
             "myhostname", then the string "{hostname}" will be replaced with
@@ -30,8 +30,8 @@ properties:
                 items:
                     type: string
                 description: |
-                    List of source directories to backup. Globs and tildes are
-                    expanded. Do not backslash spaces in path names.
+                    List of source directories and files to backup. Globs and
+                    tildes are expanded. Do not backslash spaces in path names.
                 example:
                     - /home
                     - /etc
@@ -46,29 +46,21 @@ properties:
                     properties:
                         path:
                             type: string
-                            description: |
-                                Path to local or remote repository
-                                (required). Tildes are expanded.
-                                Multiple repositories are backed up
-                                to in sequence. Borg placeholders
-                                can be used. See the output of
-                                "borg help placeholders" for
-                                details. See ssh_command for SSH
-                                options like identity file or port.
-                                If systemd service is used, then
-                                add local repository paths in the
-                                systemd service file to the
-                                ReadWritePaths list.
                             example: ssh://user@backupserver/./{fqdn}
                         label:
                             type: string
-                            description: |
-                                Optional label for the repository. This 
-                                can be used with the --repository flag 
-                                to select a repository to backup to. 
-                                If not specified, the repository path is 
-                                used as the label.
                             example: backupserver
+                description: |
+                    A required list of local or remote repositories with paths
+                    and optional labels (which can be used with the --repository
+                    flag to select a repository). Tildes are expanded. Multiple
+                    repositories are backed up to in sequence. Borg placeholders
+                    can be used. See the output of "borg help placeholders" for
+                    details. See ssh_command for SSH options like identity file
+                    or port. If systemd service is used, then add local
+                    repository paths in the systemd service file to the
+                    ReadWritePaths list. Prior to borgmatic 1.7.10, repositories
+                    was a list of plain path strings.
                 example:
                     - path: ssh://user@backupserver/./sourcehostname.borg
                       label: backupserver
@@ -386,12 +378,21 @@ properties:
                 description: |
                     Name of the archive. Borg placeholders can be used. See the
                     output of "borg help placeholders" for details. Defaults to
-                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this
-                    option, consider also specifying a prefix in the retention
-                    and consistency sections to avoid accidental
-                    pruning/checking of archives with different archive name
-                    formats.
+                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". When running
+                    actions like rlist, info, or check, borgmatic automatically
+                    tries to match only archives created with this name format.
                 example: "{hostname}-documents-{now}"
+            match_archives:
+                type: string
+                description: |
+                    A Borg pattern for filtering down the archives used by
+                    borgmatic actions that operate on multiple archives. For
+                    Borg 1.x, use a shell pattern here and see the output of
+                    "borg help placeholders" for details. For Borg 2.x, see the
+                    output of "borg help match-archives". If match_archives is
+                    not specified, borgmatic defaults to deriving the
+                    match_archives value from archive_name_format.
+                example: "sh:{hostname}-*"
             relocated_repo_access_is_ok:
                 type: boolean
                 description: |
@@ -485,10 +486,12 @@ properties:
             prefix:
                 type: string
                 description: |
-                    When pruning, only consider archive names starting with this
-                    prefix.  Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. Defaults to
-                    "{hostname}-". Use an empty value to disable the default.
+                    Deprecated. When pruning, only consider archive names
+                    starting with this prefix. Borg placeholders can be used.
+                    See the output of "borg help placeholders" for details.
+                    If a prefix is not specified, borgmatic defaults to
+                    matching archives based on the archive_name_format (see
+                    above).
                 example: sourcehostname
     consistency:
         type: object
@@ -546,12 +549,12 @@ properties:
                 items:
                     type: string
                 description: |
-                    Paths to a subset of the repositories in the location
-                    section on which to run consistency checks. Handy in case
-                    some of your repositories are very large, and so running
-                    consistency checks on them would take too long. Defaults to
-                    running consistency checks on all repositories configured in
-                    the location section.
+                    Paths or labels for a subset of the repositories in the
+                    location section on which to run consistency checks. Handy
+                    in case some of your repositories are very large, and so
+                    running consistency checks on them would take too long.
+                    Defaults to running consistency checks on all repositories
+                    configured in the location section.
                 example:
                     - user@backupserver:sourcehostname.borg
             check_last:
@@ -564,11 +567,12 @@ properties:
             prefix:
                 type: string
                 description: |
-                    When performing the "archives" check, only consider archive
-                    names starting with this prefix. Borg placeholders can be
-                    used. See the output of "borg help placeholders" for
-                    details. Defaults to "{hostname}-". Use an empty value to
-                    disable the default.
+                    Deprecated. When performing the "archives" check, only
+                    consider archive names starting with this prefix. Borg
+                    placeholders can be used. See the output of "borg help
+                    placeholders" for details. If a prefix is not specified,
+                    borgmatic defaults to matching archives based on the
+                    archive_name_format (see above).
                 example: sourcehostname
     output:
         type: object
@@ -832,25 +836,25 @@ properties:
                                 Command to use instead of "pg_dump" or
                                 "pg_dumpall". This can be used to run a specific
                                 pg_dump version (e.g., one inside a running
-                                docker container). Defaults to "pg_dump" for
-                                single database dump or "pg_dumpall" to dump
-                                all databases.
+                                container). Defaults to "pg_dump" for single
+                                database dump or "pg_dumpall" to dump all
+                                databases.
                             example: docker exec my_pg_container pg_dump
                         pg_restore_command:
                             type: string
                             description: |
                                 Command to use instead of "pg_restore". This
                                 can be used to run a specific pg_restore
-                                version (e.g., one inside a running docker
-                                container). Defaults to "pg_restore".
+                                version (e.g., one inside a running container).
+                                Defaults to "pg_restore".
                             example: docker exec my_pg_container pg_restore
                         psql_command:
                             type: string
                             description: |
                                 Command to use instead of "psql". This can be
                                 used to run a specific psql version (e.g.,
-                                one inside a running docker container).
-                                Defaults to "psql".
+                                one inside a running container). Defaults to
+                                "psql".
                             example: docker exec my_pg_container psql
                         options:
                             type: string
@@ -937,14 +941,14 @@ properties:
                             type: string
                             enum: ['sql']
                             description: |
-                                Database dump output format. Currenly only "sql"
-                                is supported. Defaults to "sql" for a single
-                                database. Or, when database name is "all" and
-                                format is blank, dumps all databases to a single
-                                file. But if a format is specified with an "all"
-                                database name, dumps each database to a separate
-                                file of that format, allowing more convenient
-                                restores of individual databases.
+                                Database dump output format. Currently only
+                                "sql" is supported. Defaults to "sql" for a
+                                single database. Or, when database name is "all"
+                                and format is blank, dumps all databases to a
+                                single file. But if a format is specified with
+                                an "all" database name, dumps each database to a
+                                separate file of that format, allowing more
+                                convenient restores of individual databases.
                             example: directory
                         add_drop_database:
                             type: boolean
@@ -1212,7 +1216,7 @@ properties:
                         type: string
                         description: |
                             Healthchecks ping URL or UUID to notify when a
-                            backup begins, ends, errors or just to send logs.
+                            backup begins, ends, errors, or to send only logs.
                         example: https://hc-ping.com/your-uuid-here
                     verify_tls:
                         type: boolean

borgmatic/config/validate.py

@@ -1,9 +1,14 @@
 import os
 
 import jsonschema
-import pkg_resources
 import ruamel.yaml
 
+try:
+    import importlib_metadata
+except ModuleNotFoundError:  # pragma: nocover
+    import importlib.metadata as importlib_metadata
+
+import borgmatic.config
 from borgmatic.config import environment, load, normalize, override
 
 
@@ -11,8 +16,19 @@ def schema_filename():
     '''
     Path to the installed YAML configuration schema file, used to validate and parse the
     configuration.
+
+    Raise FileNotFoundError when the schema path does not exist.
     '''
-    return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
+    try:
+        return next(
+            str(path.locate())
+            for path in importlib_metadata.files('borgmatic')
+            if path.match('config/schema.yaml')
+        )
+    except StopIteration:
+        # If the schema wasn't found in the package's files, this is probably a pip editable
+        # install, so try a different approach to get the schema.
+        return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml')
 
 
 def format_json_error_path_element(path_element):
@@ -69,7 +85,10 @@ def apply_logical_validation(config_filename, parsed_configuration):
     location_repositories = parsed_configuration.get('location', {}).get('repositories')
     check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
     for repository in check_repositories:
-        if repository not in location_repositories:
+        if not any(
+            repositories_match(repository, config_repository)
+            for config_repository in location_repositories
+        ):
             raise Validation_error(
                 config_filename,
                 (

borgmatic/execute.py

@@ -236,7 +236,11 @@ def execute_command(
 
 
 def execute_command_and_capture_output(
-    full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None,
+    full_command,
+    capture_stderr=False,
+    shell=False,
+    extra_environment=None,
+    working_directory=None,
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its

borgmatic/hooks/mongodb.py

@@ -161,4 +161,7 @@ def build_restore_command(extract_process, database, dump_filename):
         command.extend(('--authenticationDatabase', database['authentication_database']))
     if 'restore_options' in database:
         command.extend(database['restore_options'].split(' '))
+    if database['schemas']:
+        for schema in database['schemas']:
+            command.extend(('--nsInclude', schema))
     return command

borgmatic/hooks/mysql.py

@@ -100,7 +100,9 @@ def execute_dump_command(
     dump.create_named_pipe_for_dump(dump_filename)
 
     return execute_command(
-        dump_command, extra_environment=extra_environment, run_to_completion=False,
+        dump_command,
+        extra_environment=extra_environment,
+        run_to_completion=False,
     )
 
 

borgmatic/hooks/postgresql.py

@@ -1,6 +1,8 @@
 import csv
+import itertools
 import logging
 import os
+import shlex
 
 from borgmatic.execute import (
     execute_command,
@@ -59,8 +61,10 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
     if dry_run:
         return ()
 
+    psql_command = shlex.split(database.get('psql_command') or 'psql')
     list_command = (
-        ('psql', '--list', '--no-password', '--csv', '--tuples-only')
+        tuple(psql_command)
+        + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
@@ -122,7 +126,12 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
                 continue
 
             command = (
-                (dump_command, '--no-password', '--clean', '--if-exists',)
+                (
+                    dump_command,
+                    '--no-password',
+                    '--clean',
+                    '--if-exists',
+                )
                 + (('--host', database['hostname']) if 'hostname' in database else ())
                 + (('--port', str(database['port'])) if 'port' in database else ())
                 + (('--username', database['username']) if 'username' in database else ())
@@ -145,7 +154,9 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             if dump_format == 'directory':
                 dump.create_parent_directory_for_dump(dump_filename)
                 execute_command(
-                    command, shell=True, extra_environment=extra_environment,
+                    command,
+                    shell=True,
+                    extra_environment=extra_environment,
                 )
             else:
                 dump.create_named_pipe_for_dump(dump_filename)
@@ -202,9 +213,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(location_config), database['name'], database.get('hostname')
     )
-    psql_command = database.get('psql_command') or 'psql'
+    psql_command = shlex.split(database.get('psql_command') or 'psql')
     analyze_command = (
-        (psql_command, '--no-password', '--quiet')
+        tuple(psql_command)
+        + ('--no-password', '--no-psqlrc', '--quiet')
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
@@ -212,20 +224,25 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
         + ('--command', 'ANALYZE')
     )
-    pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
+    use_psql_command = all_databases or database.get('format') == 'plain'
+    pg_restore_command = shlex.split(database.get('pg_restore_command') or 'pg_restore')
     restore_command = (
-        (psql_command if all_databases else pg_restore_command, '--no-password')
-        + (
-            ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
-            if not all_databases
-            else ()
-        )
+        tuple(psql_command if use_psql_command else pg_restore_command)
+        + ('--no-password',)
+        + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
+        + (('--dbname', database['name']) if not all_databases else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
         + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
         + (() if extract_process else (dump_filename,))
+        + tuple(
+            itertools.chain.from_iterable(('--schema', schema) for schema in database['schemas'])
+            if database['schemas']
+            else ()
+        )
     )
+
     extra_environment = make_extra_environment(database)
 
     logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")

borgmatic/logger.py

@@ -68,7 +68,7 @@ class Multi_stream_handler(logging.Handler):
 
     def emit(self, record):
         '''
-        Dispatch the log record to the approriate stream handler for the record's log level.
+        Dispatch the log record to the appropriate stream handler for the record's log level.
         '''
         self.log_level_to_handler[record.levelno].emit(record)
 
@@ -156,6 +156,7 @@ def configure_logging(
     log_file_log_level=None,
     monitoring_log_level=None,
     log_file=None,
+    log_file_format=None,
 ):
     '''
     Configure logging to go to both the console and (syslog or log file). Use the given log levels,
@@ -200,12 +201,18 @@ def configure_logging(
 
     if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
-        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
+        syslog_handler.setFormatter(
+            logging.Formatter('borgmatic: {levelname} {message}', style='{')  # noqa: FS003
+        )
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
     elif log_file:
         file_handler = logging.handlers.WatchedFileHandler(log_file)
-        file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s'))
+        file_handler.setFormatter(
+            logging.Formatter(
+                log_file_format or '[{asctime}] {levelname}: {message}', style='{'  # noqa: FS003
+            )
+        )
         file_handler.setLevel(log_file_log_level)
         handlers = (console_handler, file_handler)
     else:

docs/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.17.1 as borgmatic
+FROM docker.io/alpine:3.17.1 as borgmatic
 
 COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
@@ -8,7 +8,7 @@ RUN borgmatic --help > /command-line.txt \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:19.5.0-alpine as html
+FROM docker.io/node:19.5.0-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -28,7 +28,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.22.1-alpine
+FROM docker.io/nginx:1.22.1-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/_includes/components/toc.css

@@ -94,7 +94,7 @@
 	display: block;
 }
 
-/* Footer catgory navigation */
+/* Footer category navigation */
 .elv-cat-list-active {
 	font-weight: 600;
 }

docs/_includes/layouts/base.njk

@@ -3,6 +3,7 @@
 	<head>
 		<meta charset="utf-8">
 		<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<link rel="icon" href="docs/static/borgmatic.png" type="image/x-icon">
 		<title>{{ subtitle + ' - ' if subtitle}}{{ title }}</title>
 {%- set css %}
 {% include 'index.css' %}

docs/docker-compose.yaml

@@ -0,0 +1,22 @@
+version: '3'
+services:
+  docs:
+    image: borgmatic-docs
+    container_name: docs
+    ports:
+      - 8080:80
+    build:
+      dockerfile: docs/Dockerfile
+      context: ..
+      args:
+        ENVIRONMENT: dev
+  message:
+    image: alpine
+    container_name: message
+    command:
+      - sh
+      - -c
+      - |
+        echo "You can view dev docs at http://localhost:8080"
+    depends_on:
+      - docs

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -66,6 +66,9 @@ variables you can use here:
 
  * `configuration_filename`: borgmatic configuration filename in which the
    hook was defined
+ * `log_file`
+   <span class="minilink minilink-addedin">New in version 1.7.12</span>:
+   path of the borgmatic log file, only set when the `--log-file` flag is used
  * `repository`: path of the current repository as configured in the current
    borgmatic configuration file
 

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -49,9 +49,12 @@ location:
         - /home
 
     repositories:
-        - /mnt/removable/backup.borg
+        - path: /mnt/removable/backup.borg
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 Then, write a `before_backup` hook in that same configuration file that uses
 the external `findmnt` utility to see whether the drive is mounted before
 proceeding.
@@ -79,13 +82,16 @@ location:
         - /home
 
     repositories:
-        - ssh://me@buddys-server.org/./backup.borg
+        - path: ssh://me@buddys-server.org/./backup.borg
 
 hooks:
     before_backup:
       - ping -q -c 1 buddys-server.org > /dev/null || exit 75
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 Or to only run backups if the battery level is high enough:
 
 ```yaml
@@ -110,8 +116,8 @@ There are some caveats you should be aware of with this feature.
  * You'll generally want to put a soft failure command in the `before_backup`
    hook, so as to gate whether the backup action occurs. While a soft failure is
    also supported in the `after_backup` hook, returning a soft failure there
-   won't prevent any actions from occuring, because they've already occurred!
-   Similiarly, you can return a soft failure from an `on_error` hook, but at
+   won't prevent any actions from occurring, because they've already occurred!
+   Similarly, you can return a soft failure from an `on_error` hook, but at
    that point it's too late to prevent the error.
  * Returning a soft failure does prevent further commands in the same hook from
    executing. So, like a standard error, it is an "early out". Unlike a standard

docs/how-to/backup-your-databases.md

@@ -138,9 +138,9 @@ hooks:
 
 ### Containers
 
-If your database is running within a Docker container and borgmatic is too, no
-problem—simply configure borgmatic to connect to the container's name on its
-exposed port. For instance:
+If your database is running within a container and borgmatic is too, no
+problem—configure borgmatic to connect to the container's name on its exposed
+port. For instance:
 
 ```yaml
 hooks:
@@ -154,10 +154,10 @@ hooks:
 
 But what if borgmatic is running on the host? You can still connect to a
 database container if its ports are properly exposed to the host. For
-instance, when running the database container with Docker, you can specify
-`--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432
-to port 5433 on the host (only reachable on localhost, in this case). Or the
-same thing with Docker Compose:
+instance, when running the database container, you can specify `--publish
+127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433
+on the host (only reachable on localhost, in this case). Or the same thing
+with Docker Compose:
 
 ```yaml
 services:
@@ -179,7 +179,7 @@ hooks:
           password: trustsome1
 ```
 
-Of course, alter the ports in these examples to suit your particular database
+You can alter the ports in these examples to suit your particular database
 system.
 
 
@@ -324,6 +324,17 @@ includes any combined dump file named "all" and any other individual database
 dumps found in the archive.
 
 
+### Restore particular schemas
+
+<span class="minilink minilink-addedin">New in version 1.7.13</span> With
+PostgreSQL and MongoDB, you can limit the restore to a single schema found
+within the database dump:
+
+```bash
+borgmatic restore --archive latest --database users --schema tentant1
+```
+
+
 ### Limitations
 
 There are a few important limitations with borgmatic's current database
@@ -386,9 +397,9 @@ dumps with any database system.
 With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors
 when borgmatic tries to connect to your database, a natural reaction is to
 increase your borgmatic verbosity with `--verbosity 2` and go looking in the
-logs. You'll notice however that your database password does not show up in
-the logs. This is likely not the cause of the authentication problem unless
-you mistyped your password, however; borgmatic passes your password to the
+logs. You'll notice though that your database password does not show up in the
+logs. This is likely not the cause of the authentication problem unless you
+mistyped your password, however; borgmatic passes your password to the
 database via an environment variable that does not appear in the logs.
 
 The cause of an authentication error is often on the database side—in the

docs/how-to/deal-with-very-large-backups.md

@@ -95,6 +95,7 @@ See [Borg's check
 documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html)
 for more information.
 
+
 ### Check frequency
 
 <span class="minilink minilink-addedin">New in version 1.6.2</span> You can

docs/how-to/develop-on-borgmatic.md

@@ -7,7 +7,7 @@ eleventyNavigation:
 ---
 ## Source code
 
-To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
+To get set up to hack on borgmatic, first clone it via HTTPS or SSH:
 
 ```bash
 git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git
@@ -25,7 +25,7 @@ so that you can run borgmatic commands while you're hacking on them to
 make sure your changes work.
 
 ```bash
-cd borgmatic/
+cd borgmatic
 pip3 install --user --editable .
 ```
 
@@ -51,7 +51,6 @@ pip3 install --user tox
 Finally, to actually run tests, run:
 
 ```bash
-cd borgmatic
 tox
 ```
 
@@ -74,24 +73,58 @@ can ask isort to order your imports for you:
 tox -e isort
 ```
 
+Similarly, if you get errors about spelling mistakes in source code, you can
+ask [codespell](https://github.com/codespell-project/codespell) to correct
+them:
+
+```bash
+tox -e codespell
+```
+
+
 ### End-to-end tests
 
 borgmatic additionally includes some end-to-end tests that integration test
 with Borg and supported databases for a few representative scenarios. These
 tests don't run by default when running `tox`, because they're relatively slow
-and depend on Docker containers for runtime dependencies. These tests tests do
-run on the continuous integration (CI) server, and running them on your
-developer machine is the closest thing to CI test parity.
+and depend on containers for runtime dependencies. These tests do run on the
+continuous integration (CI) server, and running them on your developer machine
+is the closest thing to CI-test parity.
 
-If you would like to run the full test suite, first install Docker and [Docker
-Compose](https://docs.docker.com/compose/install/). Then run:
+If you would like to run the full test suite, first install Docker (or Podman;
+see below) and [Docker Compose](https://docs.docker.com/compose/install/).
+Then run:
 
 ```bash
 scripts/run-end-to-end-dev-tests
 ```
 
-Note that this scripts assumes you have permission to run Docker. If you
-don't, then you may need to run with `sudo`.
+This script assumes you have permission to run `docker`. If you don't, then
+you may need to run with `sudo`.
+
+
+#### Podman
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span>
+borgmatic's end-to-end tests optionally support using
+[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md)
+[Podman](https://podman.io/) instead of Docker.
+
+Setting up Podman is outside the scope of this documentation, but here are
+some key points to double-check:
+
+ * Install Podman and your desired networking support.
+ * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the
+   non-root user who will run tests.
+ * Create a non-root Podman socket for that user:
+   ```bash
+   systemctl --user enable --now podman.socket
+   systemctl --user start --now podman.socket
+   ```
+
+Then you'll be able to run end-to-end tests as per normal, and the test script
+will automatically use your non-root Podman socket instead of a Docker socket.
+
 
 ## Code style
 
@@ -101,10 +134,10 @@ the following deviations from it:
  * For strings, prefer single quotes over double quotes.
  * Limit all lines to a maximum of 100 characters.
  * Use trailing commas within multiline values or argument lists.
- * For multiline constructs, put opening and closing delimeters on lines
+ * For multiline constructs, put opening and closing delimiters on lines
    separate from their contents.
  * Within multiline constructs, use standard four-space indentation. Don't align
-   indentation with an opening delimeter.
+   indentation with an opening delimiter.
 
 borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
 formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
@@ -129,11 +162,12 @@ To build and view a copy of the documentation with your local changes, run the
 following from the root of borgmatic's source code:
 
 ```bash
-sudo scripts/dev-docs
+scripts/dev-docs
 ```
 
-This requires Docker to be installed on your system. You may not need to use
-sudo if your non-root user has permissions to run Docker.
+This requires Docker (or Podman; see below) to be installed on your system.
+This script assumes you have permission to run `docker`. If you don't, then
+you may need to run with `sudo`.
 
 After you run the script, you can point your web browser at
 http://localhost:8080 to view the documentation with your changes.
@@ -141,3 +175,15 @@ http://localhost:8080 to view the documentation with your changes.
 To close the documentation server, ctrl-C the script. Note that it does not
 currently auto-reload, so you'll need to stop it and re-run it for any
 additional documentation changes to take effect.
+
+
+#### Podman
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span>
+borgmatic's developer build for documentation optionally supports using
+[rootless](https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md)
+[Podman](https://podman.io/) instead of Docker.
+
+Setting up Podman is outside the scope of this documentation. But once you
+install and configure Podman, then `scripts/dev-docs` should automatically use
+Podman instead of Docker.

docs/how-to/inspect-your-backups.md

@@ -111,7 +111,7 @@ By default, borgmatic logs to a local syslog-compatible daemon if one is
 present and borgmatic is running in a non-interactive console. Where those
 logs show up depends on your particular system. If you're using systemd, try
 running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or
-similiar.
+similar.
 
 You can customize the log level used for syslog logging with the
 `--syslog-verbosity` flag, and this is independent from the console logging
@@ -154,5 +154,39 @@ borgmatic --log-file /path/to/file.log
 
 Note that if you use the `--log-file` flag, you are responsible for rotating
 the log file so it doesn't grow too large, for example with
-[logrotate](https://wiki.archlinux.org/index.php/Logrotate). Also, there is a
-`--log-file-verbosity` flag to customize the log file's log level.
+[logrotate](https://wiki.archlinux.org/index.php/Logrotate).
+
+You can the `--log-file-verbosity` flag to customize the log file's log level:
+
+```bash
+borgmatic --log-file /path/to/file.log --log-file-verbosity 2
+```
+
+<span class="minilink minilink-addedin">New in version 1.7.11</span> Use the
+`--log-file-format` flag to override the default log message format. This
+format string can contain a series of named placeholders wrapped in curly
+brackets. For instance, the default log format is: `[{asctime}] {levelname}:
+{message}`. This means each log message is recorded as the log time (in square
+brackets), a logging level name, a colon, and the actual log message.
+
+So if you only want each log message to get logged *without* a timestamp or a
+logging level name:
+
+```bash
+borgmatic --log-file /path/to/file.log --log-file-format "{message}"
+```
+
+Here is a list of available placeholders:
+
+ * `{asctime}`: time the log message was created
+ * `{levelname}`: level of the log message (`INFO`, `DEBUG`, etc.)
+ * `{lineno}`: line number in the source file where the log message originated
+ * `{message}`: actual log message
+ * `{pathname}`: path of the source file where the log message originated
+
+See the [Python logging
+documentation](https://docs.python.org/3/library/logging.html#logrecord-attributes)
+for additional placeholders.
+
+Note that this `--log-file-format` flg only applies to the specified
+`--log-file` and not to syslog or other logging.

docs/how-to/make-backups-redundant.md

@@ -24,6 +24,9 @@ location:
         - path: /var/lib/backups/local.borg
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 When you run borgmatic with this configuration, it invokes Borg once for each
 configured repository in sequence. (So, not in parallel.) That means—in each
 repository—borgmatic creates a single new backup archive containing all of
@@ -31,9 +34,8 @@ your source directories.
 
 Here's a way of visualizing what borgmatic does with the above configuration:
 
-1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg`
-2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
-3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
+1. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
+2. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
 
 This gives you redundancy of your data across repositories and even
 potentially across providers.

docs/how-to/make-per-application-backups.md

@@ -54,6 +54,96 @@ choice](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot),
 each entry using borgmatic's `--config` flag instead of relying on
 `/etc/borgmatic.d`.
 
+
+## Archive naming
+
+If you've got multiple borgmatic configuration files, you might want to create
+archives with different naming schemes for each one. This is especially handy
+if each configuration file is backing up to the same Borg repository but you
+still want to be able to distinguish backup archives for one application from
+another.
+
+borgmatic supports this use case with an `archive_name_format` option. The
+idea is that you define a string format containing a number of [Borg
+placeholders](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-placeholders),
+and borgmatic uses that format to name any new archive it creates. For
+instance:
+
+```yaml
+storage:
+    ...
+    archive_name_format: home-directories-{now}
+```
+
+This means that when borgmatic creates an archive, its name will start with
+the string `home-directories-` and end with a timestamp for its creation time.
+If `archive_name_format` is unspecified, the default is
+`{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a
+timestamp in a particular format.
+
+
+### Archive filtering
+
+<span class="minilink minilink-addedin">New in version 1.7.11</span> borgmatic
+uses the `archive_name_format` option to automatically limit which archives
+get used for actions operating on multiple archives. This prevents, for
+instance, duplicate archives from showing up in `rlist` or `info` results—even
+if the same repository appears in multiple borgmatic configuration files. To
+take advantage of this feature, use a different `archive_name_format` in each
+configuration file.
+
+Under the hood, borgmatic accomplishes this by substituting globs for certain
+ephemeral data placeholders in your `archive_name_format`—and using the result
+to filter archives when running supported actions.
+
+For instance, let's say that you have this in your configuration:
+
+```yaml
+storage:
+    ...
+    archive_name_format: {hostname}-user-data-{now}
+```
+
+borgmatic considers `{now}` an emphemeral data placeholder that will probably
+change per archive, while `{hostname}` won't. So it turns the example value
+into `{hostname}-user-data-*` and applies it to filter down the set of
+archives used for actions like `rlist`, `info`, `prune`, `check`, etc.
+
+The end result is that when borgmatic runs the actions for a particular
+application-specific configuration file, it only operates on the archives
+created for that application. But this doesn't apply to actions like `compact`
+that operate on an entire repository.
+
+If this behavior isn't quite smart enough for your needs, you can use the
+`match_archives` option to override the pattern that borgmatic uses for
+filtering archives. For example:
+
+```yaml
+storage:
+    ...
+    archive_name_format: {hostname}-user-data-{now}
+    match_archives: sh:myhost-user-data-*        
+```
+
+For Borg 1.x, use a shell pattern for the `match_archives` value and see the
+[Borg patterns
+documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns)
+for more information. For Borg 2.x, see the [match archives
+documentation](https://borgbackup.readthedocs.io/en/2.0.0b5/usage/help.html#borg-help-match-archives).
+
+Some borgmatic command-line actions also have a `--match-archives` flag that
+overrides both the auto-matching behavior and the `match_archives`
+configuration option.
+
+<span class="minilink minilink-addedin">Prior to 1.7.11</span> The way to
+limit the archives used for the `prune` action was a `prefix` option in the
+`retention` section for matching against the start of archive names. And the
+option for limiting the archives used for the `check` action was a separate
+`prefix` in the `consistency` section. Both of these options are deprecated in
+favor of the auto-matching behavior (or `match_archives`/`--match-archives`)
+in newer versions of borgmatic.
+
+
 ## Configuration includes
 
 Once you have multiple different configuration files, you might want to share
@@ -185,9 +275,140 @@ Once this include gets merged in, the resulting configuration would have a
 When there's an option collision between the local file and the merged
 include, the local file's option takes precedence.
 
+
+#### List merge
+
 <span class="minilink minilink-addedin">New in version 1.6.1</span> Colliding
 list values are appended together.
 
+<span class="minilink minilink-addedin">New in version 1.7.12</span> If there
+is a list value from an include that you *don't* want in your local
+configuration file, you can omit it with an `!omit` tag. For instance:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   source_directories:
+     - !omit /home
+     - /var
+```
+
+And `common.yaml` like this:
+
+```yaml
+location:
+   source_directories:
+     - /home
+     - /etc
+```
+
+Once this include gets merged in, the resulting configuration will have a
+`source_directories` value of `/etc` and `/var`—with `/home` omitted.
+
+This feature currently only works on scalar (e.g. string or number) list items
+and will not work elsewhere in a configuration file. Be sure to put the
+`!omit` tag *before* the list item (after the dash). Putting `!omit` after the
+list item will not work, as it gets interpreted as part of the string. Here's
+an example of some things not to do:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   source_directories:
+     # Do not do this! It will not work. "!omit" belongs before "/home".
+     - /home !omit
+
+   # Do not do this either! "!omit" only works on scalar list items.
+   repositories: !omit
+     # Also do not do this for the same reason! This is a list item, but it's
+     # not a scalar.
+     - !omit path: repo.borg
+```
+
+Additionally, the `!omit` tag only works in a configuration file that also
+performs a merge include with `<<: !include`. It doesn't make sense within,
+for instance, an included configuration file itself (unless it in turn
+performs its own merge include). That's because `!omit` only applies to the
+file doing the include; it doesn't work in reverse or propagate through
+includes.
+
+
+### Shallow merge
+
+Even though deep merging is generally pretty handy for included files,
+sometimes you want specific sections in the local file to take precedence over
+included sections—without any merging occurring for them.
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span> That's
+where the `!retain` tag comes in. Whenever you're merging an included file
+into your configuration file, you can optionally add the `!retain` tag to
+particular local mappings or lists to retain the local values and ignore
+included values.
+
+For instance, start with this configuration file containing the `!retain` tag
+on the `retention` mapping:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   repositories:
+     - path: repo.borg
+
+retention: !retain
+    keep_daily: 5
+```
+
+And `common.yaml` like this:
+
+```yaml
+location:
+   repositories:
+     - path: common.borg
+
+retention:
+    keep_hourly: 24
+    keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration will have a
+`keep_daily` value of `5` and nothing else in the `retention` section. That's
+because the `!retain` tag says to retain the local version of `retention` and
+ignore any values coming in from the include. But because the `repositories`
+list doesn't have a `!retain` tag, it still gets merged together to contain
+both `common.borg` and `repo.borg`.
+
+The `!retain` tag can only be placed on mappings and lists, and it goes right
+after the name of the option (and its colon) on the same line. The effects of
+`!retain` are recursive, meaning that if you place a `!retain` tag on a
+top-level mapping, even deeply nested values within it will not be merged.
+
+Additionally, the `!retain` tag only works in a configuration file that also
+performs a merge include with `<<: !include`. It doesn't make sense within,
+for instance, an included configuration file itself (unless it in turn
+performs its own merge include). That's because `!retain` only applies to the
+file doing the include; it doesn't work in reverse or propagate through
+includes.
+
+
+## Debugging includes
+
+<span class="minilink minilink-addedin">New in version 1.7.12</span> If you'd
+like to see what the loaded configuration looks like after includes get merged
+in, run `validate-borgmatic-config` on your configuration file:
+
+```bash
+sudo validate-borgmatic-config --show
+```
+
+You'll need to specify your configuration file with `--config` if it's not in
+a default location.
+
+This will output the merged configuration as borgmatic sees it, which can be
+helpful for understanding how your includes work in practice.
+
 
 ## Configuration overrides
 
@@ -272,7 +493,7 @@ Here's an example usage:
 ```yaml
 constants:
     user: foo
-    my_prefix: bar-
+    archive_prefix: bar
 
 location:
     source_directories:
@@ -281,20 +502,14 @@ location:
     ...
 
 storage:
-    archive_name_format: '{my_prefix}{now}'
-
-retention:
-    prefix: {my_prefix}
-
-consistency:
-    prefix: {my_prefix}
+    archive_name_format: '{archive_prefix}-{now}'
 ```
 
 In this example, when borgmatic runs, all instances of `{user}` get replaced
-with `foo` and all instances of `{my_prefix}` get replaced with `bar-`. (And
-in this particular example, `{now}` doesn't get replaced with anything, but
-gets passed directly to Borg.) After substitution, the logical result looks
-something like this:
+with `foo` and all instances of `{archive-prefix}` get replaced with `bar-`.
+(And in this particular example, `{now}` doesn't get replaced with anything,
+but gets passed directly to Borg.) After substitution, the logical result
+looks something like this:
 
 ```yaml
 location:
@@ -305,12 +520,6 @@ location:
 
 storage:
     archive_name_format: 'bar-{now}'
-
-retention:
-    prefix: bar-
-
-consistency:
-    prefix: bar-
 ```
 
 An alternate to constants is passing in your values via [environment

docs/how-to/monitor-your-backups.md

@@ -73,7 +73,7 @@ from borgmatic for a configured interval.
 
 ### Consistency checks
 
-While not strictly part of monitoring, if you really want confidence that your
+While not strictly part of monitoring, if you want confidence that your
 backups are not only running but are restorable as well, you can configure
 particular [consistency
 checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration)

docs/how-to/run-arbitrary-borg-commands.md

@@ -89,8 +89,10 @@ borgmatic's `borg` action is not without limitations:
  * Unlike normal borgmatic actions that support JSON, the `borg` action will
    not disable certain borgmatic logs to avoid interfering with JSON output.
  * Unlike other borgmatic actions, the `borg` action captures (and logs) all
-   output, so interactive prompts or flags like `--progress` will not work as
-   expected.
+   output, so interactive prompts and flags like `--progress` will not work as
+   expected. <span class="minilink minilink-addedin">New in version
+   1.7.13</span> borgmatic now runs the `borg` action without capturing output,
+   so interactive prompts work.
 
 In general, this `borgmatic borg` feature should be considered an escape
 valve—a feature of second resort. In the long run, it's preferable to wrap

docs/how-to/set-up-backups.md

@@ -82,21 +82,22 @@ on a relatively dedicated system, then a global install can work out fine.
 Besides the approaches described above, there are several other options for
 installing borgmatic:
 
- * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files)
- * [Docker image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/)
+ * [container image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files)
+ * [container image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/)
  * [Debian](https://tracker.debian.org/pkg/borgmatic)
  * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
  * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
  * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/)
+ * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic)
  * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
  * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
- * [OpenBSD](http://ports.su/sysutils/borgmatic)
+ * [OpenBSD](https://openports.pl/path/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
  * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic)
  * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/)
  * [NixOS](https://search.nixos.org/packages?show=borgmatic&sort=relevance&type=packages&query=borgmatic)
  * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup)
- * [virtualenv](https://virtualenv.pypa.io/en/stable/)
+ * [Unraid](https://unraid.net/community/apps?q=borgmatic#r)
 
 
 ## Hosting providers
@@ -157,7 +158,7 @@ variable or set the `BORG_PASSPHRASE` environment variable. See the
 section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)
 of the Borg Quick Start for more info.
 
-Alternatively, you can specify the passphrase programatically by setting
+Alternatively, you can specify the passphrase programmatically by setting
 either the borgmatic `encryption_passcommand` configuration variable or the
 `BORG_PASSCOMMAND` environment variable. See the [Borg Security
 FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
@@ -180,6 +181,9 @@ following command is available for that:
 sudo validate-borgmatic-config
 ```
 
+You'll need to specify your configuration file with `--config` if it's not in
+a default location.
+
 This command's exit status (`$?` in Bash) is zero when configuration is valid
 and non-zero otherwise.
 
@@ -276,7 +280,7 @@ that, you can configure a separate job runner to invoke it periodically.
 ### cron
 
 If you're using cron, download the [sample cron
-file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/master/sample/cron/borgmatic).
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/main/sample/cron/borgmatic).
 Then, from the directory where you downloaded it:
 
 ```bash
@@ -300,9 +304,9 @@ you may already have borgmatic systemd service and timer files. If so, you may
 be able to skip some of the steps below.)
 
 First, download the [sample systemd service
-file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.service)
 and the [sample systemd timer
-file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.timer).
 
 Then, from the directory where you downloaded them:
 
@@ -331,10 +335,13 @@ Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293).
 
 ### Shell completion
 
-borgmatic includes a shell completion script (currently only for Bash) to
+borgmatic includes a shell completion script (currently only for Bash and Fish) to
 support tab-completing borgmatic command-line actions and flags. Depending on
-how you installed borgmatic, this may be enabled by default. But if it's not,
-start by installing the `bash-completion` Linux package or the
+how you installed borgmatic, this may be enabled by default.
+
+#### Bash
+
+If completions aren't enabled, start by installing the `bash-completion` Linux package or the
 [`bash-completion@2`](https://formulae.brew.sh/formula/bash-completion@2)
 macOS Homebrew formula. Then, install the shell completion script globally:
 
@@ -359,6 +366,14 @@ borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmat
 Finally, restart your shell (`exit` and open a new shell) so the completions
 take effect.
 
+#### fish
+
+To add completions for fish, install the completions file globally:
+
+```fish
+borgmatic --fish-completion | sudo tee /usr/share/fish/vendor_completions.d/borgmatic.fish
+source /usr/share/fish/vendor_completions.d/borgmatic.fish
+```
 
 ### Colored output
 

docs/how-to/upgrade.md

@@ -145,15 +145,18 @@ like this:
 ```yaml
 location:
     repositories:
-        - original.borg
+        - path: original.borg
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.10</span> Omit
+the `path:` portion of the `repositories` list.
+
 Change it to a new (not yet created) repository path:
 
 ```yaml
 location:
     repositories:
-        - upgraded.borg
+        - path: upgraded.borg
 ```
 
 Then, run the `rcreate` action (formerly `init`) to create that new Borg 2

docs/reference/command-line.md

@@ -7,8 +7,10 @@ eleventyNavigation:
 ---
 ## borgmatic options
 
-Here are all of the available borgmatic command-line options. This includes the separate options for
-each action sub-command:
+Here are all of the available borgmatic command-line options, including the
+separate options for each action sub-command. Note that most of the
+flags listed here do not have equivalents in borgmatic's [configuration
+file](https://torsion.org/borgmatic/docs/reference/configuration/).
 
 ```
 {% include borgmatic/command-line.txt %}

scripts/dev-docs

@@ -2,8 +2,10 @@
 
 set -e
 
-docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile .
-echo
-echo "You can view dev docs at http://localhost:8080"
-echo
-docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs
+USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock
+
+if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then
+    export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
+fi
+
+docker-compose --file docs/docker-compose.yaml up --build --force-recreate

scripts/run-end-to-end-dev-tests

@@ -1,7 +1,7 @@
 #!/bin/sh
 
-# This script is for running all tests, including end-to-end tests, on a developer machine. It sets
-# up database containers to run tests against, runs the tests, and then tears down the containers.
+# This script is for running end-to-end tests on a developer machine. It sets up database containers
+# to run tests against, runs the tests, and then tears down the containers.
 #
 # Run this script from the root directory of the borgmatic source.
 #
@@ -10,5 +10,11 @@
 
 set -e
 
+USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock
+
+if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then
+    export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
+fi
+
 docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
     --renew-anon-volumes --abort-on-container-exit

scripts/run-full-tests

@@ -8,17 +8,24 @@
 # For more information, see:
 # https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
 
-set -ex
+set -e
+
+if [ -z "$TEST_CONTAINER" ]; then
+    echo "This script is designed to work inside a test container and is not intended to"
+    echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute"
+    echo "scripts/run-end-to-end-dev-tests instead."
+    exit 1
+fi
 
 apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
-    py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite
+    py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite fish
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
 python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1
 pip3 install --ignore-installed tox==3.25.1
 export COVERAGE_FILE=/tmp/.coverage
 
-if [ "$1" != "--end-to-end-only" ] ; then
+if [ "$1" != "--end-to-end-only" ]; then
     tox --workdir /tmp/.tox --sitepackages
 fi
 

setup.cfg

@@ -4,19 +4,23 @@ description_file=README.md
 [tool:pytest]
 testpaths = tests
 addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end
-filterwarnings = 
-    ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning
 
 [flake8]
-ignore = E501,W503
+max-line-length = 100
+extend-ignore = E203,E501,W503
 exclude = *.*/*
 multiline-quotes = '''
 docstring-quotes = '''
 
 [tool:isort]
-force_single_line = False
-include_trailing_comma = True
+profile=black
 known_first_party = borgmatic
 line_length = 100
-multi_line_output = 3
 skip = .tox
+
+[codespell]
+skip = .git,.tox,build
+
+[pycodestyle]
+ignore = E203
+max_line_length = 100

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.7.10'
+VERSION = '1.7.13'
 
 
 setup(
@@ -32,6 +32,7 @@ setup(
     install_requires=(
         'colorama>=0.4.1,<0.5',
         'jsonschema',
+        'packaging',
         'requests',
         'ruamel.yaml>0.15.0,<0.18.0',
         'setuptools',

test_requirements.txt

@@ -1,26 +1,33 @@
 appdirs==1.4.4; python_version >= '3.8'
-attrs==20.3.0; python_version >= '3.8'
-black==19.10b0; python_version >= '3.8'
-click==7.1.2; python_version >= '3.8'
-colorama==0.4.4
-coverage==5.3
-flake8==4.0.1
+attrs==22.2.0; python_version >= '3.8'
+black==23.3.0; python_version >= '3.8'
+chardet==5.1.0
+click==8.1.3; python_version >= '3.8'
+codespell==2.2.4
+colorama==0.4.6
+coverage==7.2.3
+flake8==6.0.0
 flake8-quotes==3.3.2
 flake8-use-fstring==1.4
 flake8-variables-names==0.0.5
-flexmock==0.10.4
-isort==5.9.1
-mccabe==0.6.1
-pluggy==0.13.1
-pathspec==0.8.1; python_version >= '3.8'
-py==1.10.0
-pycodestyle==2.8.0
-pyflakes==2.4.0
-jsonschema==3.2.0
-pytest==7.2.0
+flexmock==0.11.3
+idna==3.4
+importlib_metadata==6.3.0; python_version < '3.8'
+isort==5.12.0
+mccabe==0.7.0
+packaging==23.1
+pluggy==1.0.0
+pathspec==0.11.1; python_version >= '3.8'
+py==1.11.0
+pycodestyle==2.10.0
+pyflakes==3.0.1
+jsonschema==4.17.3
+pytest==7.3.0
 pytest-cov==4.0.0
 regex; python_version >= '3.8'
-requests==2.25.0
+requests==2.28.2
 ruamel.yaml>0.15.0,<0.18.0
 toml==0.10.2; python_version >= '3.8'
 typed-ast; python_version >= '3.8'
+typing-extensions==4.5.0; python_version < '3.8'
+zipp==3.15.0; python_version < '3.8'

tests/end-to-end/docker-compose.yaml

@@ -1,26 +1,29 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:13.1-alpine
+    image: docker.io/postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   mysql:
-    image: mariadb:10.5
+    image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   mongodb:
-    image: mongo:5.0.5
+    image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
   tests:
-    image: alpine:3.13
+    image: docker.io/alpine:3.13
+    environment:
+      TEST_CONTAINER: true
     volumes:
       - "../..:/app:ro"
     tmpfs:
       - "/app/borgmatic.egg-info"
+      - "/app/build"
     tty: true
     working_dir: /app
     entrypoint: /app/scripts/run-full-tests
@@ -28,3 +31,4 @@ services:
     depends_on:
       - postgresql
       - mysql
+      - mongodb

tests/end-to-end/test_completion.py

@@ -3,3 +3,7 @@ import subprocess
 
 def test_bash_completion_runs_without_error():
     subprocess.check_call('borgmatic --bash-completion | bash', shell=True)
+
+
+def test_fish_completion_runs_without_error():
+    subprocess.check_call('borgmatic --fish-completion | fish', shell=True)

tests/end-to-end/test_database.py

@@ -118,7 +118,7 @@ def test_database_dump_and_restore():
 
         # Restore the database from the archive.
         subprocess.check_call(
-            ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name]
+            ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
         )
     finally:
         os.chdir(original_working_directory)

tests/end-to-end/test_validate_config.py

@@ -1,5 +1,6 @@
 import os
 import subprocess
+import sys
 import tempfile
 
 
@@ -26,3 +27,16 @@ def test_validate_config_command_with_invalid_configuration_fails():
         exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' '))
 
         assert exit_code == 1
+
+
+def test_validate_config_command_with_show_flag_displays_configuration():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        output = subprocess.check_output(
+            f'validate-borgmatic-config --config {config_path} --show'.split(' ')
+        ).decode(sys.stdout.encoding)
+
+        assert 'location:' in output
+        assert 'repositories:' in output

tests/integration/borg/test_commands.py

@@ -0,0 +1,126 @@
+import argparse
+import copy
+
+from flexmock import flexmock
+
+import borgmatic.borg.info
+import borgmatic.borg.list
+import borgmatic.borg.rlist
+import borgmatic.borg.transfer
+import borgmatic.commands.arguments
+
+
+def assert_command_does_not_duplicate_flags(command, *args, **kwargs):
+    '''
+    Assert that the given Borg command sequence does not contain any duplicated flags, e.g.
+    "--match-archives" twice anywhere in the command.
+    '''
+    flag_counts = {}
+
+    for flag_name in command:
+        if not flag_name.startswith('--'):
+            continue
+
+        if flag_name in flag_counts:
+            flag_counts[flag_name] += 1
+        else:
+            flag_counts[flag_name] = 1
+
+    assert flag_counts == {
+        flag_name: 1 for flag_name in flag_counts
+    }, f"Duplicate flags found in: {' '.join(command)}"
+
+
+def fuzz_argument(arguments, argument_name):
+    '''
+    Given an argparse.Namespace instance of arguments and an argument name in it, copy the arguments
+    namespace and set the argument name in the copy with a fake value. Return the copied arguments.
+
+    This is useful for "fuzzing" a unit under test by passing it each possible argument in turn,
+    making sure it doesn't blow up or duplicate Borg arguments.
+    '''
+    arguments_copy = copy.copy(arguments)
+    value = getattr(arguments_copy, argument_name)
+    setattr(arguments_copy, argument_name, not value if isinstance(value, bool) else 'value')
+
+    return arguments_copy
+
+
+def test_transfer_archives_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments(
+        'transfer', '--source-repository', 'foo'
+    )['transfer']
+    flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with(
+        assert_command_does_not_duplicate_flags
+    )
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        borgmatic.borg.transfer.transfer_archives(
+            False,
+            'repo',
+            {},
+            '2.3.4',
+            fuzz_argument(arguments, argument_name),
+            global_arguments=flexmock(log_json=False),
+        )
+
+
+def test_make_list_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments('list')['list']
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        command = borgmatic.borg.list.make_list_command(
+            'repo',
+            {},
+            '2.3.4',
+            fuzz_argument(arguments, argument_name),
+            argparse.Namespace(log_json=False),
+        )
+
+        assert_command_does_not_duplicate_flags(command)
+
+
+def test_make_rlist_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments('rlist')['rlist']
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        command = borgmatic.borg.rlist.make_rlist_command(
+            'repo',
+            {},
+            '2.3.4',
+            fuzz_argument(arguments, argument_name),
+            global_arguments=flexmock(log_json=True),
+        )
+
+        assert_command_does_not_duplicate_flags(command)
+
+
+def test_display_archives_info_command_does_not_duplicate_flags_or_raise():
+    arguments = borgmatic.commands.arguments.parse_arguments('info')['info']
+    flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with(
+        assert_command_does_not_duplicate_flags
+    )
+    flexmock(borgmatic.borg.info).should_receive('execute_command').replace_with(
+        assert_command_does_not_duplicate_flags
+    )
+
+    for argument_name in dir(arguments):
+        if argument_name.startswith('_'):
+            continue
+
+        borgmatic.borg.info.display_archives_info(
+            'repo',
+            {},
+            '2.3.4',
+            fuzz_argument(arguments, argument_name),
+            argparse.Namespace(log_json=False),
+        )

tests/integration/commands/test_arguments.py

@@ -422,6 +422,13 @@ def test_parse_arguments_disallows_list_with_progress_for_create_action():
         module.parse_arguments('create', '--list', '--progress')
 
 
+def test_parse_arguments_disallows_list_with_json_for_create_action():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('create', '--list', '--json')
+
+
 def test_parse_arguments_allows_json_with_list_or_info():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
@@ -465,6 +472,20 @@ def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives
         )
 
 
+def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar')
+
+
+def test_parse_arguments_disallows_rlist_with_both_prefix_and_match_archives():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('rlist', '--prefix', 'foo', '--match-archives', 'sh:*bar')
+
+
 def test_parse_arguments_disallows_info_with_both_archive_and_match_archives():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/commands/test_completion.py

@@ -3,3 +3,7 @@ from borgmatic.commands import completion as module
 
 def test_bash_completion_does_not_raise():
     assert module.bash_completion()
+
+
+def test_fish_completion_does_not_raise():
+    assert module.fish_completion()

tests/integration/commands/test_validate_config.py

@@ -18,3 +18,12 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list():
     parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
 
     assert parser.config_paths == ['myconfig', 'otherconfig']
+
+
+def test_parse_arguments_supports_show_flag():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments('--config', 'myconfig', '--show')
+
+    assert parser.show

tests/integration/config/test_load.py

@@ -2,7 +2,6 @@ import io
 import sys
 
 import pytest
-import ruamel.yaml
 from flexmock import flexmock
 
 from borgmatic.config import load as module
@@ -150,6 +149,248 @@ def test_load_configuration_merges_include():
     assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
 
 
+def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          foo: bar
+          baz: quux
+
+        other:
+          a: b
+          c: d
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff: !retain
+          foo: override
+
+        other:
+          a: override
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {
+        'stuff': {'foo': 'override'},
+        'other': {'a': 'override', 'c': 'd'},
+    }
+
+
+def test_load_configuration_with_retain_tag_but_without_merge_include_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff: !retain
+          foo: bar
+          baz: quux
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          foo: override
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_retain_tag_on_scalar_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          foo: bar
+          baz: quux
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          foo: !retain override
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_merges_include_and_omits_requested_values():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - !omit b
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'stuff': ['a', 'c', 'x', 'y']}
+
+
+def test_load_configuration_with_omit_tag_on_unknown_value_merges_include_and_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - !omit q
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'stuff': ['a', 'b', 'c', 'x', 'y']}
+
+
+def test_load_configuration_with_omit_tag_on_non_list_item_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff: !omit
+          - x
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_on_non_scalar_list_item_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - foo: bar
+            baz: quux
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - !omit foo: bar
+            baz: quux
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
+def test_load_configuration_with_omit_tag_but_without_merge_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include_file = io.StringIO(
+        '''
+        stuff:
+          - a
+          - !omit b
+          - c
+        '''
+    )
+    include_file.name = 'include.yaml'
+    builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file)
+    config_file = io.StringIO(
+        '''
+        stuff:
+          - x
+          - y
+        <<: !include include.yaml
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
 def test_load_configuration_does_not_merge_include_list():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -173,42 +414,79 @@ def test_load_configuration_does_not_merge_include_list():
     config_file.name = 'config.yaml'
     builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
 
-    with pytest.raises(ruamel.yaml.error.YAMLError):
+    with pytest.raises(module.ruamel.yaml.error.YAMLError):
         assert module.load_configuration('config.yaml')
 
 
+@pytest.mark.parametrize(
+    'node_class',
+    (
+        module.ruamel.yaml.nodes.MappingNode,
+        module.ruamel.yaml.nodes.SequenceNode,
+        module.ruamel.yaml.nodes.ScalarNode,
+    ),
+)
+def test_raise_retain_node_error_raises(node_class):
+    with pytest.raises(ValueError):
+        module.raise_retain_node_error(
+            loader=flexmock(), node=node_class(tag=flexmock(), value=flexmock())
+        )
+
+
+def test_raise_omit_node_error_raises():
+    with pytest.raises(ValueError):
+        module.raise_omit_node_error(loader=flexmock(), node=flexmock())
+
+
+def test_filter_omitted_nodes():
+    nodes = [
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+    ]
+
+    result = module.filter_omitted_nodes(nodes)
+
+    assert [item.value for item in result] == ['a', 'c', 'a', 'c']
+
+
 def test_deep_merge_nodes_replaces_colliding_scalar_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_hourly'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
                     ),
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_daily'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
                     ),
                 ],
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_daily'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
                     ),
                 ],
             ),
@@ -230,35 +508,39 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values():
 def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_hourly'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
                     ),
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_daily'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
                     ),
                 ],
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='keep_minutely'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='10'),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='10'
+                        ),
                     ),
                 ],
             ),
@@ -282,28 +564,28 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
 def test_deep_merge_nodes_keeps_deeply_nested_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='lock_wait'
                         ),
-                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
                     ),
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='extra_borg_options'
                         ),
-                        ruamel.yaml.nodes.MappingNode(
+                        module.ruamel.yaml.nodes.MappingNode(
                             tag='tag:yaml.org,2002:map',
                             value=[
                                 (
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='init'
                                     ),
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='--init-option'
                                     ),
                                 ),
@@ -314,22 +596,22 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='extra_borg_options'
                         ),
-                        ruamel.yaml.nodes.MappingNode(
+                        module.ruamel.yaml.nodes.MappingNode(
                             tag='tag:yaml.org,2002:map',
                             value=[
                                 (
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='prune'
                                     ),
-                                    ruamel.yaml.nodes.ScalarNode(
+                                    module.ruamel.yaml.nodes.ScalarNode(
                                         tag='tag:yaml.org,2002:str', value='--prune-option'
                                     ),
                                 ),
@@ -361,32 +643,48 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
 def test_deep_merge_nodes_appends_colliding_sequence_values():
     node_values = [
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
-                        ruamel.yaml.nodes.SequenceNode(
-                            tag='tag:yaml.org,2002:int', value=['echo 1', 'echo 2']
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
                         ),
                     ),
                 ],
             ),
         ),
         (
-            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
-            ruamel.yaml.nodes.MappingNode(
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
                 tag='tag:yaml.org,2002:map',
                 value=[
                     (
-                        ruamel.yaml.nodes.ScalarNode(
+                        module.ruamel.yaml.nodes.ScalarNode(
                             tag='tag:yaml.org,2002:str', value='before_backup'
                         ),
-                        ruamel.yaml.nodes.SequenceNode(
-                            tag='tag:yaml.org,2002:int', value=['echo 3', 'echo 4']
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 4'
+                                ),
+                            ],
                         ),
                     ),
                 ],
@@ -401,4 +699,178 @@ def test_deep_merge_nodes_appends_colliding_sequence_values():
     options = section_value.value
     assert len(options) == 1
     assert options[0][0].value == 'before_backup'
-    assert options[0][1].value == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
+    assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 2', 'echo 3', 'echo 4']
+
+
+def test_deep_merge_nodes_only_keeps_mapping_values_tagged_with_retain():
+    node_values = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_hourly'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
+                    ),
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='!retain',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'retention'
+    assert section_value.tag == 'tag:yaml.org,2002:map'
+    options = section_value.value
+    assert len(options) == 1
+    assert options[0][0].value == 'keep_daily'
+    assert options[0][1].value == '5'
+
+
+def test_deep_merge_nodes_only_keeps_sequence_values_tagged_with_retain():
+    node_values = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='!retain',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 4'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'hooks'
+    options = section_value.value
+    assert len(options) == 1
+    assert options[0][0].value == 'before_backup'
+    assert options[0][1].tag == 'tag:yaml.org,2002:seq'
+    assert [item.value for item in options[0][1].value] == ['echo 3', 'echo 4']
+
+
+def test_deep_merge_nodes_skips_sequence_values_tagged_with_omit():
+    node_values = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 1'
+                                ),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 2'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='hooks'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='before_backup'
+                        ),
+                        module.ruamel.yaml.nodes.SequenceNode(
+                            tag='tag:yaml.org,2002:seq',
+                            value=[
+                                module.ruamel.yaml.ScalarNode(tag='!omit', value='echo 2'),
+                                module.ruamel.yaml.ScalarNode(
+                                    tag='tag:yaml.org,2002:str', value='echo 3'
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'hooks'
+    options = section_value.value
+    assert len(options) == 1
+    assert options[0][0].value == 'before_backup'
+    assert [item.value for item in options[0][1].value] == ['echo 1', 'echo 3']

tests/integration/config/test_validate.py

@@ -8,7 +8,7 @@ from flexmock import flexmock
 from borgmatic.config import validate as module
 
 
-def test_schema_filename_returns_plausable_path():
+def test_schema_filename_returns_plausible_path():
     schema_path = module.schema_filename()
 
     assert schema_path.endswith('/schema.yaml')

tests/integration/test_execute.py

@@ -147,7 +147,7 @@ def test_log_outputs_kills_other_processes_when_one_errors():
         ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
     )
     flexmock(module).should_receive('exit_code_indicates_error').with_args(
-        other_process, None, 'borg'
+        ['sleep', '2'], None, 'borg'
     ).and_return(False)
     flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
         process.stdout

tests/unit/actions/test_borg.py

@@ -16,6 +16,7 @@ def test_run_borg_does_not_raise():
         repository={'path': 'repos'},
         storage={},
         local_borg_version=None,
+        global_arguments=flexmock(log_json=False),
         borg_arguments=borg_arguments,
         local_path=None,
         remote_path=None,

tests/unit/actions/test_break_lock.py

@@ -14,6 +14,7 @@ def test_run_break_lock_does_not_raise():
         storage={},
         local_borg_version=None,
         break_lock_arguments=break_lock_arguments,
+        global_arguments=flexmock(),
         local_path=None,
         remote_path=None,
     )

tests/unit/actions/test_check.py

@@ -12,7 +12,11 @@ def test_run_check_calls_hooks_for_configured_repository():
     flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
     flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
     check_arguments = flexmock(
-        repository=None, progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock(),
+        repository=None,
+        progress=flexmock(),
+        repair=flexmock(),
+        only=flexmock(),
+        force=flexmock(),
     )
     global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
 

tests/unit/actions/test_info.py

@@ -18,6 +18,7 @@ def test_run_info_does_not_raise():
             storage={},
             local_borg_version=None,
             info_arguments=info_arguments,
+            global_arguments=flexmock(log_json=False),
             local_path=None,
             remote_path=None,
         )

tests/unit/actions/test_list.py

@@ -18,6 +18,7 @@ def test_run_list_does_not_raise():
             storage={},
             local_borg_version=None,
             list_arguments=list_arguments,
+            global_arguments=flexmock(log_json=False),
             local_path=None,
             remote_path=None,
         )

tests/unit/actions/test_mount.py

@@ -21,6 +21,7 @@ def test_run_mount_does_not_raise():
         storage={},
         local_borg_version=None,
         mount_arguments=mount_arguments,
+        global_arguments=flexmock(log_json=False),
         local_path=None,
         remote_path=None,
     )

tests/unit/actions/test_restore.py

@@ -72,6 +72,7 @@ def test_collect_archive_database_names_parses_archive_paths():
         location={'borgmatic_source_directory': '.borgmatic'},
         storage=flexmock(),
         local_borg_version=flexmock(),
+        global_arguments=flexmock(log_json=False),
         local_path=flexmock(),
         remote_path=flexmock(),
     )
@@ -97,6 +98,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths():
         location={'borgmatic_source_directory': '.borgmatic'},
         storage=flexmock(),
         local_borg_version=flexmock(),
+        global_arguments=flexmock(log_json=False),
         local_path=flexmock(),
         remote_path=flexmock(),
     )
@@ -118,6 +120,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths():
         location={'borgmatic_source_directory': '.borgmatic'},
         storage=flexmock(),
         local_borg_version=flexmock(),
+        global_arguments=flexmock(log_json=False),
         local_path=flexmock(),
         remote_path=flexmock(),
     )
@@ -148,7 +151,8 @@ def test_find_databases_to_restore_without_requested_names_finds_all_archive_dat
     archive_database_names = {'postresql_databases': ['foo', 'bar']}
 
     restore_names = module.find_databases_to_restore(
-        requested_database_names=[], archive_database_names=archive_database_names,
+        requested_database_names=[],
+        archive_database_names=archive_database_names,
     )
 
     assert restore_names == archive_database_names
@@ -158,7 +162,8 @@ def test_find_databases_to_restore_with_all_in_requested_names_finds_all_archive
     archive_database_names = {'postresql_databases': ['foo', 'bar']}
 
     restore_names = module.find_databases_to_restore(
-        requested_database_names=['all'], archive_database_names=archive_database_names,
+        requested_database_names=['all'],
+        archive_database_names=archive_database_names,
     )
 
     assert restore_names == archive_database_names
@@ -194,7 +199,9 @@ def test_ensure_databases_found_with_all_databases_found_does_not_raise():
 def test_ensure_databases_found_with_no_databases_raises():
     with pytest.raises(ValueError):
         module.ensure_databases_found(
-            restore_names={'postgresql_databases': []}, remaining_restore_names={}, found_names=[],
+            restore_names={'postgresql_databases': []},
+            remaining_restore_names={},
+            found_names=[],
         )
 
 
@@ -233,7 +240,7 @@ def test_run_restore_restores_each_database():
         remote_path=object,
         archive_name=object,
         hook_name='postgresql_databases',
-        database={'name': 'foo'},
+        database={'name': 'foo', 'schemas': None},
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -246,7 +253,7 @@ def test_run_restore_restores_each_database():
         remote_path=object,
         archive_name=object,
         hook_name='postgresql_databases',
-        database={'name': 'bar'},
+        database={'name': 'bar', 'schemas': None},
     ).once()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -256,7 +263,9 @@ def test_run_restore_restores_each_database():
         storage=flexmock(),
         hooks=flexmock(),
         local_borg_version=flexmock(),
-        restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()),
+        restore_arguments=flexmock(
+            repository='repo', archive='archive', databases=flexmock(), schemas=None
+        ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
         remote_path=flexmock(),
@@ -327,7 +336,7 @@ def test_run_restore_restores_database_configured_with_all_name():
         remote_path=object,
         archive_name=object,
         hook_name='postgresql_databases',
-        database={'name': 'foo'},
+        database={'name': 'foo', 'schemas': None},
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -340,7 +349,7 @@ def test_run_restore_restores_database_configured_with_all_name():
         remote_path=object,
         archive_name=object,
         hook_name='postgresql_databases',
-        database={'name': 'bar'},
+        database={'name': 'bar', 'schemas': None},
     ).once()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -350,7 +359,9 @@ def test_run_restore_restores_database_configured_with_all_name():
         storage=flexmock(),
         hooks=flexmock(),
         local_borg_version=flexmock(),
-        restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()),
+        restore_arguments=flexmock(
+            repository='repo', archive='archive', databases=flexmock(), schemas=None
+        ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
         remote_path=flexmock(),
@@ -399,7 +410,7 @@ def test_run_restore_skips_missing_database():
         remote_path=object,
         archive_name=object,
         hook_name='postgresql_databases',
-        database={'name': 'foo'},
+        database={'name': 'foo', 'schemas': None},
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -412,7 +423,7 @@ def test_run_restore_skips_missing_database():
         remote_path=object,
         archive_name=object,
         hook_name='postgresql_databases',
-        database={'name': 'bar'},
+        database={'name': 'bar', 'schemas': None},
     ).never()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -422,7 +433,9 @@ def test_run_restore_skips_missing_database():
         storage=flexmock(),
         hooks=flexmock(),
         local_borg_version=flexmock(),
-        restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()),
+        restore_arguments=flexmock(
+            repository='repo', archive='archive', databases=flexmock(), schemas=None
+        ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
         remote_path=flexmock(),
@@ -465,7 +478,7 @@ def test_run_restore_restores_databases_from_different_hooks():
         remote_path=object,
         archive_name=object,
         hook_name='postgresql_databases',
-        database={'name': 'foo'},
+        database={'name': 'foo', 'schemas': None},
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -478,7 +491,7 @@ def test_run_restore_restores_databases_from_different_hooks():
         remote_path=object,
         archive_name=object,
         hook_name='mysql_databases',
-        database={'name': 'bar'},
+        database={'name': 'bar', 'schemas': None},
     ).once()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -488,7 +501,9 @@ def test_run_restore_restores_databases_from_different_hooks():
         storage=flexmock(),
         hooks=flexmock(),
         local_borg_version=flexmock(),
-        restore_arguments=flexmock(repository='repo', archive='archive', databases=flexmock()),
+        restore_arguments=flexmock(
+            repository='repo', archive='archive', databases=flexmock(), schemas=None
+        ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
         remote_path=flexmock(),

tests/unit/actions/test_rinfo.py

@@ -15,6 +15,7 @@ def test_run_rinfo_does_not_raise():
             storage={},
             local_borg_version=None,
             rinfo_arguments=rinfo_arguments,
+            global_arguments=flexmock(log_json=False),
             local_path=None,
             remote_path=None,
         )

tests/unit/actions/test_rlist.py

@@ -15,6 +15,7 @@ def test_run_rlist_does_not_raise():
             storage={},
             local_borg_version=None,
             rlist_arguments=rlist_arguments,
+            global_arguments=flexmock(),
             local_path=None,
             remote_path=None,
         )

tests/unit/actions/test_transfer.py

@@ -10,7 +10,7 @@ def test_run_transfer_does_not_raise():
     global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
 
     module.run_transfer(
-        repository='repo',
+        repository={'path': 'repo'},
         storage={},
         local_borg_version=None,
         transfer_arguments=transfer_arguments,

tests/unit/borg/test_borg.py

@@ -7,7 +7,7 @@ from borgmatic.borg import borg as module
 from ..test_verbosity import insert_logging_mock
 
 
-def test_run_arbitrary_borg_calls_borg_with_parameters():
+def test_run_arbitrary_borg_calls_borg_with_flags():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@@ -15,7 +15,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -28,7 +28,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters():
     )
 
 
-def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
+def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@@ -36,7 +36,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--info'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -50,7 +50,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
     )
 
 
-def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
+def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@@ -58,7 +58,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--debug', '--show-rc'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -72,7 +72,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
     )
 
 
-def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters():
+def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     storage_config = {'lock_wait': 5}
@@ -83,7 +83,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters(
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--lock-wait', '5'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -96,7 +96,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters(
     )
 
 
-def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter():
+def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
@@ -106,7 +106,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo::archive'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -128,7 +128,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'break-lock', 'repo'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg1',
         extra_environment=None,
     )
@@ -142,7 +142,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
     )
 
 
-def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters():
+def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@@ -152,7 +152,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -166,7 +166,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet
     )
 
 
-def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
+def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@@ -174,7 +174,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo', '--progress'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -187,7 +187,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
     )
 
 
-def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
+def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@@ -195,7 +195,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'break-lock', 'repo'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -208,7 +208,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
     )
 
 
-def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
+def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
     flexmock(module.flags).should_receive('make_repository_flags').never()
@@ -216,13 +216,16 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg',),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
 
     module.run_arbitrary_borg(
-        repository_path='repo', storage_config={}, local_borg_version='1.2.3', options=[],
+        repository_path='repo',
+        storage_config={},
+        local_borg_version='1.2.3',
+        options=[],
     )
 
 
@@ -234,7 +237,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository():
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'key', 'export', 'repo'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -255,7 +258,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository()
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'debug', 'dump-manifest', 'repo', 'path'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -276,7 +279,7 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'debug', 'info'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )
@@ -297,7 +300,7 @@ def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_bor
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'debug', 'convert-profile', 'in', 'out'),
-        output_log_level=module.borgmatic.logger.ANSWER,
+        output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         borg_local_path='borg',
         extra_environment=None,
     )

tests/unit/borg/test_break_lock.py

@@ -10,7 +10,9 @@ from ..test_verbosity import insert_logging_mock
 def insert_execute_command_mock(command):
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        command, borg_local_path='borg', extra_environment=None,
+        command,
+        borg_local_path='borg',
+        extra_environment=None,
     ).once()
 
 
@@ -19,7 +21,10 @@ def test_break_lock_calls_borg_with_required_flags():
     insert_execute_command_mock(('borg', 'break-lock', 'repo'))
 
     module.break_lock(
-        repository_path='repo', storage_config={}, local_borg_version='1.2.3',
+        repository_path='repo',
+        storage_config={},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False),
     )
 
 
@@ -28,7 +33,11 @@ def test_break_lock_calls_borg_with_remote_path_flags():
     insert_execute_command_mock(('borg', 'break-lock', '--remote-path', 'borg1', 'repo'))
 
     module.break_lock(
-        repository_path='repo', storage_config={}, local_borg_version='1.2.3', remote_path='borg1',
+        repository_path='repo',
+        storage_config={},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False),
+        remote_path='borg1',
     )
 
 
@@ -37,7 +46,22 @@ def test_break_lock_calls_borg_with_umask_flags():
     insert_execute_command_mock(('borg', 'break-lock', '--umask', '0770', 'repo'))
 
     module.break_lock(
-        repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3',
+        repository_path='repo',
+        storage_config={'umask': '0770'},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False),
+    )
+
+
+def test_break_lock_calls_borg_with_log_json_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    insert_execute_command_mock(('borg', 'break-lock', '--log-json', 'repo'))
+
+    module.break_lock(
+        repository_path='repo',
+        storage_config={},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=True),
     )
 
 
@@ -46,7 +70,10 @@ def test_break_lock_calls_borg_with_lock_wait_flags():
     insert_execute_command_mock(('borg', 'break-lock', '--lock-wait', '5', 'repo'))
 
     module.break_lock(
-        repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3',
+        repository_path='repo',
+        storage_config={'lock_wait': '5'},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False),
     )
 
 
@@ -56,7 +83,10 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter():
     insert_logging_mock(logging.INFO)
 
     module.break_lock(
-        repository_path='repo', storage_config={}, local_borg_version='1.2.3',
+        repository_path='repo',
+        storage_config={},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False),
     )
 
 
@@ -66,5 +96,8 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags():
     insert_logging_mock(logging.DEBUG)
 
     module.break_lock(
-        repository_path='repo', storage_config={}, local_borg_version='1.2.3',
+        repository_path='repo',
+        storage_config={},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False),
     )