Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1206

.bandit

@@ -0,0 +1,3 @@
+[bandit]
+exclude=tests
+skips=S105,S404

.eleventy.js

@@ -21,7 +21,7 @@ module.exports = function(eleventyConfig) {
             if (process.env.NODE_ENV == "production") {
                 return link;
             }
-            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
+            return link.replace('https://torsion.org/', 'http://localhost:8080/');
         }
     };
     let markdownItAnchorOptions = {
@@ -44,7 +44,7 @@ module.exports = function(eleventyConfig) {
         templateFormats: [
           "md",
           "txt"
-        ]
+        ],
     }
 };
 

.gitea/pull_request_template.md

@@ -0,0 +1,4 @@
+---
+name: "Pull Request"
+about: "Pull Request"
+---

.gitea/workflows/build.yaml

@@ -29,5 +29,3 @@ jobs:
           PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
       - run: podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
       - run: podman push "$IMAGE_NAME"
-      - run: scripts/export-docs-from-image
-      - run: curl --user "${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}" --upload-file borgmatic-docs.tar.gz https://projects.torsion.org/api/packages/borgmatic-collective/generic/borgmatic-docs/$(head --lines=1 NEWS)/borgmatic-docs.tar.gz

.github/pull_request_template.md

@@ -1,6 +1,8 @@
-## Hold up!
+## Hold up, GitHub users
 
-Thanks for your contribution. Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see:
+Thanks for your contribution! 
+
+Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see:
 
 https://torsion.org/borgmatic/#contributing
 

MANIFEST.in

@@ -1,2 +1,7 @@
+# This file only applies to the source dist tarball, not the built wheel.
 include borgmatic/config/schema.yaml
-graft sample/systemd
+graft docs
+graft sample
+graft scripts
+graft tests
+global-exclude *.py[co]

NEWS

@@ -1,3 +1,149 @@
+2.0.13
+ * #1054: Allow the Btrfs hook to create and delete snapshots even when running
+   as a non-root user. See the documentation for more information:
+   https://torsion.org/borgmatic/reference/configuration/data-sources/btrfs/#non-root-user
+ * #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always
+   error and exit when the borgmatic runtime directory overlaps with the configured excludes.
+ * #1179: Add a "file_list_format" option for setting the "list" action's output format and an
+   "archive_list_format" option for setting the "repo-list" action's format.
+ * #1192: Fix for over-aggressive deduplication of source directories that contain the borgmatic
+   runtime directory, potentially resulting in data loss (data not getting backed up) when
+   snapshotting these source directories.
+ * #1192, #1163: Document potential interactions between security settings in borgmatic's sample
+   systemd service file and the ZFS, LVM, and Btrfs hooks.
+ * #1193: In the documentation for the MariaDB/MySQL database hooks, clarify how to set custom
+   command-line flags for database commands.
+ * #1193: For the MariaDB and MySQL database hooks, add a "socket_path" option for Unix socket
+   database connections.
+ * #1193: For the MariaDB and MySQL database hooks, creates a consistent snapshot by dumping all
+   tables in a single transaction.
+ * #1194: Fix for an incorrect diff command shown when running the "generate config" action with a
+   source configuration file. 
+ * #1195: Fix a regression in the ZFS, LVM, and Btrfs hooks in which snapshotted paths ignored
+   global excludes.
+ * #1201: Document a problematic interaction between borgmatic and systemd-tmpfiles:
+   https://torsion.org/borgmatic/reference/configuration/runtime-directory/#systemd-tmpfiles
+ * #1203: Fix that errors and exits when the borgmatic runtime directory is partially excluded by
+   configured excludes. Previously, borgmatic only errored when the runtime directory was completely
+   excluded.
+ * #1206: Adjust Btrfs snapshot paths so that Borg 1.x gets file cache hits when backing them up,
+   improving performance.
+ * Update the sample systemd timer with a shorter random delay when catching up on a missed run.
+
+2.0.12
+ * #1127: Fix for the database hooks not respecting the "working_directory" option.
+ * #1181: Add an "ask_for_password" option to the KeePassXC credential hook for disabling
+   KeePassXC's password prompt, e.g. if you're only using a key file to decrypt your database.
+ * #1184: Fix the fish shell completion's detection of version mismatches.
+ * #1186: Fix a regression in the Btrfs hook in which subvolume snapshots didn't get cleaned up
+   until the start of the next borgmatic run.
+ * In the SQLite database hook, run SQLite such that it exits upon encountering an error instead of,
+   you know, not doing that.
+ * Add documentation on repositories, including SSH, Rclone, S3, and B2:
+   https://torsion.org/borgmatic/reference/configuration/repositories/
+ * Improve documentation search results for individual configuration options.
+ * Add borgmatic release artifacts (wheel and tarball) to each release on the releases page:
+   https://projects.torsion.org/borgmatic-collective/borgmatic/releases
+ * Move the tarball of borgmatic's HTML documentation from the packages page to the releases page.
+
+2.0.11
+ * #957: Document borgmatic's limitations around parallelism—both its own and Borg's. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/how-to/make-per-application-backups/#limitations
+ * #1165: Fix for when the systemd service directories (RuntimeDirectory and StateDirectory) each
+   contain multiple paths.
+ * #1168: Fix for the "list", "info", and "delete" options in "extra_borg_options" being ignored
+   when "--archive" is omitted with Borg 1.x.
+ * #1169: Fix for a regression in the ZFS, LVM, and Btrfs hooks in which partial excludes of
+   snapshot paths were ignored.
+ * #1170: Fix for an inconsistent log level for Borg's last output line before exiting.
+ * #1172: Add an "environment" option to the Sentry monitoring hook.
+ * #1176: Fix the "--repository" flag not applying to command hooks.
+ * Add a "rename" option to "extra_borg_options" to support passing arbitrary flags to "borg
+   rename".
+ * Add documentation on patterns and excludes:
+   https://torsion.org/borgmatic/reference/configuration/patterns-and-excludes/
+ * Drop support for Python 3.9, which has been end-of-lifed.
+
+2.0.10
+ * #427: Expand the "extra_borg_options" option to support passing arbitrary Borg flags to every
+   Borg sub-command that borgmatic uses. As part of this, deprecate the "init" option under
+   "borg_extra_options" in favor of "repo_create".
+ * #942: Factor reference material out of the documentation how-to guides. This means there's now a
+   whole reference section in the docs! Check it out: https://torsion.org/borgmatic/
+ * #973: For the MariaDB and MySQL database hooks, add a "skip_names" option to ignore particular
+   databases when dumping "all".
+ * #1150: Fix for a runtime directory error when the "create" action is used with the "--log-json"
+   flag.
+ * #1150: Fix for a runtime directory error when the configured patterns contain a global exclude.
+ * #1161: Fix a traceback (TypeError) in the "check" action with Python 3.14.
+ * #1166: Add a "borg_key_file" option for setting the Borg repository key file path.
+ * Add documentation search.
+ * Change the URL of the local documentation development server to be more like the production URL.
+
+2.0.9
+ * #1105: More accurately collect Btrfs subvolumes to snapshot. As part of this, the Btrfs hook no
+   longer uses "findmnt" and the "findmnt_command" option is deprecated.
+ * #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd
+   service.
+ * #1149: Add support for Python 3.14.
+ * #1149: Include automated tests in the source dist tarball uploaded to PyPI.
+ * #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+
+   "slashdot" hack within source directory paths.
+ * #1152: Fix a regression in the Loki monitoring hook in which log messages weren't sending.
+ * #1156: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to snapshot both parent and child
+   volumes/filesystems instead of just the parent. As part of this fix, borgmatic no longer
+   deduplicates patterns except for those containing the borgmatic runtime directory.
+ * Fix a traceback (TypeError) regression in the "spot" check when the "local_path" option isn't
+   set.
+
+2.0.8
+ * #1114: Document systemd configuration changes for the ZFS filesystem hook.
+ * #1116: Add dumping of database containers via their container names, handy for backing up
+   database containers from the host. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-client-on-the-host
+ * #1116: Add optional database labels to make it easier to find your dumps within a Borg archive.
+ * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in
+   use.
+ * #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always
+   error and exit when the borgmatic runtime directory overlaps with the configured excludes.
+ * #1125: Clarify documentation about ZFS, Btrfs, and LVM snapshotting when a separate
+   filesystem is mounted in the source directory. (Spoiler: The separate filesystem doesn't get
+   included in the snapshot.)
+ * #1126: Create LVM snapshots as read-write to avoid an error when snapshotting ext4 filesystems
+   with orphaned files that need recovery.
+ * #1133: Fix the "spot" check to include borgmatic configuration files that were backed up to
+   support the "bootstrap" action.
+ * #1136: For all database hooks, record metadata about the dumps contained within an archive.
+ * #1139: Set "borgmatic" as the user agent when connecting to monitoring services.
+ * #1146: Fix a broken "create" action and "--archive latest" flag when multiple archives share the
+   same name with Borg 2.
+ * Treat configuration file permissions issues as errors instead of warnings.
+ * When running tests, use Ruff for faster and more comprehensive code linting and formatting,
+   replacing Flake8, Black, isort, etc.
+ * Switch from pipx to uv for installing development tools, and added tox-uv for speeding up test
+   environment creation. See the developer documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
+
+2.0.7
+ * #1032: Fix a bug in which a Borg archive gets created even when a database hook fails.
+ * #1043: Support Btrfs subvolume paths in "source_directories" even when the subvolume is mounted
+   elsewhere.
+ * #1048: Ignore Btrfs subvolumes whose read-only status can't be determined.
+ * #1083: Add "debug_passphrase"/"display_passphrase" options and a "{unixtime}" placeholder in
+   support of Borg 2 features.
+ * #1099: Clarify documentation on command hooks order of execution.
+ * #1100: Fix a bug in which "borg --version" failing isn't considered a "fail" state in a command
+   hook.
+ * #1108: Fix a bug in which quoted "extra_borg_options" values containing spaces are passed to Borg
+   incorrectly.
+ * #1108: Add a "--comment" flag to the "create" action for creating an archive with a comment.
+ * Use the Bandit security analysis tool when running tests.
+ * SECURITY: Add timeouts to all monitoring hooks to prevent hangs on network requests, e.g. due to
+   a compromised monitoring server holding requests open.
+ * SECURITY: For the "spot" check, use a more secure source of randomness when selecting paths to
+   check.
+
 2.0.6
  * #1068: Fix a warning from LVM about leaked file descriptors.
  * #1086: Fix for the "spot" check breaking when the "--progress" flag is used.

README.md

@@ -2,10 +2,7 @@
 title: borgmatic
 permalink: index.html
 ---
-
-## It's your data. Keep it that way.
-
-<img src="docs/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
+<img src="https://torsion.org/borgmatic/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
 
 borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
@@ -60,45 +57,45 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
 ### Data
 
-<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://openzfs.org/"><img src="docs/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://btrfs.readthedocs.io/"><img src="docs/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sourceware.org/lvm2/"><img src="docs/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://rclone.org"><img src="docs/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.postgresql.org/"><img src="https://torsion.org/borgmatic/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.mysql.com/"><img src="https://torsion.org/borgmatic/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://mariadb.com/"><img src="https://torsion.org/borgmatic/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.mongodb.com/"><img src="https://torsion.org/borgmatic/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sqlite.org/"><img src="https://torsion.org/borgmatic/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://openzfs.org/"><img src="https://torsion.org/borgmatic/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://btrfs.readthedocs.io/"><img src="https://torsion.org/borgmatic/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sourceware.org/lvm2/"><img src="https://torsion.org/borgmatic/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://rclone.org"><img src="https://torsion.org/borgmatic/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="https://torsion.org/borgmatic/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ### Monitoring
 
-<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://uptime.kuma.pet/"><img src="docs/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.pushover.net/"><img src="docs/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://healthchecks.io/"><img src="https://torsion.org/borgmatic/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://uptime.kuma.pet/"><img src="https://torsion.org/borgmatic/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://cronitor.io/"><img src="https://torsion.org/borgmatic/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://cronhub.io/"><img src="https://torsion.org/borgmatic/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.pagerduty.com/"><img src="https://torsion.org/borgmatic/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.pushover.net/"><img src="https://torsion.org/borgmatic/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://ntfy.sh/"><img src="https://torsion.org/borgmatic/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://grafana.com/oss/loki/"><img src="https://torsion.org/borgmatic/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://github.com/caronc/apprise/wiki"><img src="https://torsion.org/borgmatic/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.zabbix.com/"><img src="https://torsion.org/borgmatic/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sentry.io/"><img src="https://torsion.org/borgmatic/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ### Credentials
 
-<a href="https://systemd.io/"><img src="docs/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.docker.com/"><img src="docs/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://podman.io/"><img src="docs/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://keepassxc.org/"><img src="docs/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://systemd.io/"><img src="https://torsion.org/borgmatic/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.docker.com/"><img src="https://torsion.org/borgmatic/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://podman.io/"><img src="https://torsion.org/borgmatic/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://keepassxc.org/"><img src="https://torsion.org/borgmatic/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ## Getting started
 
 Your first step is to [install and configure
-borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/).
+borgmatic](https://torsion.org/borgmatic/how-to/set-up-backups/).
 
 For additional documentation, check out the links above (left panel on wide screens)
 for <a href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
@@ -132,7 +129,7 @@ first. If you prefer to use an existing GitHub account, you can skip account
 creation and [login directly](https://projects.torsion.org/user/login).
 
 Also see the [security
-policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
+policy](https://torsion.org/borgmatic/security-policy/) for any security
 issues.
 
 
@@ -177,7 +174,7 @@ discuss your idea. Note that you'll need to
 first. In general, contributions are very welcome. We don't bite!
 
 Also, please check out the [borgmatic development
-how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
+how-to](https://torsion.org/borgmatic/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
 ### Recent contributors
@@ -186,4 +183,4 @@ Thanks to all borgmatic contributors! There are multiple ways to contribute to
 this project, so the following includes those who have fixed bugs, contributed
 features, *or* filed tickets.
 
-{% include borgmatic/contributors.html %}
+{% include borgmatic/contributors.html %}

borgmatic/actions/borg.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.borg.borg
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -19,25 +18,22 @@ def run_borg(
     '''
     Run the "borg" action for the given repository.
     '''
-    if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, borg_arguments.repository
-    ):
-        logger.info('Running arbitrary Borg command')
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            borg_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        borgmatic.borg.borg.run_arbitrary_borg(
-            repository['path'],
-            config,
-            local_borg_version,
-            options=borg_arguments.options,
-            archive=archive_name,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
+    logger.info('Running arbitrary Borg command')
+    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        borg_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    borgmatic.borg.borg.run_arbitrary_borg(
+        repository['path'],
+        config,
+        local_borg_version,
+        options=borg_arguments.options,
+        archive=archive_name,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/break_lock.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.break_lock
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,15 +17,12 @@ def run_break_lock(
     '''
     Run the "break-lock" action for the given repository.
     '''
-    if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, break_lock_arguments.repository
-    ):
-        logger.info('Breaking repository and cache locks')
-        borgmatic.borg.break_lock.break_lock(
-            repository['path'],
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
+    logger.info('Breaking repository and cache locks')
+    borgmatic.borg.break_lock.break_lock(
+        repository['path'],
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/change_passphrase.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.change_passphrase
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,19 +17,13 @@ def run_change_passphrase(
     '''
     Run the "key change-passphrase" action for the given repository.
     '''
-    if (
-        change_passphrase_arguments.repository is None
-        or borgmatic.config.validate.repositories_match(
-            repository, change_passphrase_arguments.repository
-        )
-    ):
-        logger.info('Changing repository passphrase')
-        borgmatic.borg.change_passphrase.change_passphrase(
-            repository['path'],
-            config,
-            local_borg_version,
-            change_passphrase_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
+    logger.info('Changing repository passphrase')
+    borgmatic.borg.change_passphrase.change_passphrase(
+        repository['path'],
+        config,
+        local_borg_version,
+        change_passphrase_arguments,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/check.py

@@ -1,4 +1,5 @@
 import calendar
+import contextlib
 import datetime
 import hashlib
 import itertools
@@ -10,18 +11,17 @@ import shlex
 import shutil
 import textwrap
 
+import borgmatic.actions.config.bootstrap
 import borgmatic.actions.pattern
 import borgmatic.borg.check
 import borgmatic.borg.create
 import borgmatic.borg.environment
 import borgmatic.borg.extract
 import borgmatic.borg.list
+import borgmatic.borg.pattern
 import borgmatic.borg.repo_list
-import borgmatic.borg.state
 import borgmatic.config.paths
-import borgmatic.config.validate
 import borgmatic.execute
-import borgmatic.hooks.command
 
 DEFAULT_CHECKS = (
     {'name': 'repository', 'frequency': '1 month'},
@@ -55,12 +55,14 @@ def parse_checks(config, only_checks=None):
 
     if 'disabled' in checks:
         logger.warning(
-            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
+            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead',
         )
+
         if len(checks) > 1:
             logger.warning(
-                'Multiple checks are configured, but one of them is "disabled"; not running any checks'
+                'Multiple checks are configured, but one of them is "disabled"; not running any checks',
             )
+
         return ()
 
     return checks
@@ -164,6 +166,7 @@ def filter_checks_on_frequency(
                     **dict.fromkeys(day for day in days if day != 'weekday'),
                     **dict.fromkeys(WEEKDAY_DAYS),
                 }
+
             if 'weekend' in days:
                 days = {
                     **dict.fromkeys(day for day in days if day != 'weekend'),
@@ -172,7 +175,7 @@ def filter_checks_on_frequency(
 
             if calendar.day_name[datetime_now().weekday()] not in days:
                 logger.info(
-                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)"
+                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)",
                 )
                 filtered_checks.remove(check)
                 continue
@@ -190,7 +193,7 @@ def filter_checks_on_frequency(
         if datetime_now() < check_time + frequency_delta:
             remaining = check_time + frequency_delta - datetime_now()
             logger.info(
-                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
+                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)',
             )
             filtered_checks.remove(check)
 
@@ -216,7 +219,7 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
     '''
     borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
 
-    if check_type in ('archives', 'data'):
+    if check_type in {'archives', 'data'}:
         return os.path.join(
             borgmatic_state_directory,
             'checks',
@@ -240,7 +243,7 @@ def write_check_time(path):  # pragma: no cover
     logger.debug(f'Writing check time at {path}')
 
     os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-    pathlib.Path(path, mode=0o600).touch()
+    pathlib.Path(path).touch(mode=0o600)
 
 
 def read_check_time(path):
@@ -251,7 +254,7 @@ def read_check_time(path):
     logger.debug(f'Reading check time from {path}')
 
     try:
-        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
+        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)  # noqa: DTZ006
     except FileNotFoundError:
         return None
 
@@ -282,7 +285,7 @@ def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
             (
                 make_check_time_path(config, borg_repository_id, check, archives_check_id),
                 make_check_time_path(config, borg_repository_id, check),
-            )
+            ),
         )
     )
 
@@ -314,16 +317,17 @@ def upgrade_check_times(config, borg_repository_id):
       {borgmatic_state_directory}/checks/1234567890/archives/all
     '''
     borgmatic_source_checks_path = os.path.join(
-        borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
+        borgmatic.config.paths.get_borgmatic_source_directory(config),
+        'checks',
     )
     borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
     borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
 
     if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
-        borgmatic_state_checks_path
+        borgmatic_state_checks_path,
     ):
         logger.debug(
-            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
+            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}',
         )
         os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
         shutil.move(borgmatic_source_checks_path, borgmatic_state_checks_path)
@@ -338,10 +342,8 @@ def upgrade_check_times(config, borg_repository_id):
 
         logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
 
-        try:
+        with contextlib.suppress(FileNotFoundError):
             shutil.move(old_path, temporary_path)
-        except FileNotFoundError:
-            pass
 
         os.mkdir(old_path)
         shutil.move(temporary_path, new_path)
@@ -355,41 +357,51 @@ def collect_spot_check_source_paths(
     local_path,
     remote_path,
     borgmatic_runtime_directory,
+    bootstrap_config_paths,
 ):
     '''
     Given a repository configuration dict, a configuration dict, the local Borg version, global
-    arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
-    collect the source paths that Borg would use in an actual create (but only include files).
+    arguments as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the
+    bootstrap configuration paths as read from an archive's manifest, collect the source paths that
+    Borg would use in an actual create (but only include files). As part of this, include the
+    bootstrap configuration paths, so that any configuration files included in the archive to
+    support bootstrapping are also spot checked.
     '''
     stream_processes = any(
         borgmatic.hooks.dispatch.call_hooks(
             'use_streaming',
             config,
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-        ).values()
+        ).values(),
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, pattern_file) = (
-        borgmatic.borg.create.make_base_create_command(
-            dry_run=True,
-            repository_path=repository['path'],
-            # Omit "progress" because it interferes with "list_details".
-            config=dict(
-                {option: value for option, value in config.items() if option != 'progress'},
-                list_details=True,
+    (create_flags, create_positional_arguments, _) = borgmatic.borg.create.make_base_create_command(
+        dry_run=True,
+        repository_path=repository['path'],
+        # Omit "progress" because it interferes with "list_details".
+        config=dict(
+            {option: value for option, value in config.items() if option != 'progress'},
+            list_details=True,
+        ),
+        patterns=borgmatic.actions.pattern.process_patterns(
+            borgmatic.actions.pattern.collect_patterns(config)
+            + tuple(
+                borgmatic.borg.pattern.Pattern(
+                    config_path,
+                    source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
+                )
+                for config_path in bootstrap_config_paths
             ),
-            patterns=borgmatic.actions.pattern.process_patterns(
-                borgmatic.actions.pattern.collect_patterns(config),
-                working_directory,
-            ),
-            local_borg_version=local_borg_version,
-            global_arguments=global_arguments,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
-            local_path=local_path,
-            remote_path=remote_path,
-            stream_processes=stream_processes,
-        )
+            config,
+            working_directory,
+        ),
+        local_borg_version=local_borg_version,
+        global_arguments=global_arguments,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
+        local_path=local_path,
+        remote_path=remote_path,
+        stream_processes=stream_processes,
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
@@ -405,7 +417,7 @@ def collect_spot_check_source_paths(
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.splitlines()
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
+        if path_line and path_line.startswith(('- ', '+ '))
     )
 
     return tuple(
@@ -446,12 +458,12 @@ def collect_spot_check_archive_paths(
             config,
             local_borg_version,
             global_arguments,
-            path_format='{type} {path}{NUL}',  # noqa: FS003
+            path_format='{type} {path}{NUL}',
             local_path=local_path,
             remote_path=remote_path,
         )
         for (file_type, path) in (line.split(' ', 1),)
-        if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
+        if file_type not in {BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE}
         if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
         if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
         not in pathlib.Path(path).parents
@@ -484,9 +496,10 @@ def compare_spot_check_hashes(
     # source directories.
     spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
     sample_count = max(
-        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
+        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)),
+        1,
     )
-    source_sample_paths = tuple(random.sample(source_paths, sample_count))
+    source_sample_paths = tuple(random.SystemRandom().sample(source_paths, sample_count))
     working_directory = borgmatic.config.paths.get_working_directory(config)
     hashable_source_sample_path = {
         source_path
@@ -496,7 +509,7 @@ def compare_spot_check_hashes(
         if not os.path.islink(full_source_path)
     }
     logger.debug(
-        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
+        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check',
     )
 
     source_sample_paths_iterator = iter(source_sample_paths)
@@ -508,7 +521,7 @@ def compare_spot_check_hashes(
     while True:
         # Hash each file in the sample paths (if it exists).
         source_sample_paths_subset = tuple(
-            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT)
+            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT),
         )
         if not source_sample_paths_subset:
             break
@@ -535,7 +548,7 @@ def compare_spot_check_hashes(
                     for path in source_sample_paths_subset
                     if path not in hashable_source_sample_path
                 },
-            )
+            ),
         )
 
         # Get the hash for each file in the archive.
@@ -549,12 +562,12 @@ def compare_spot_check_hashes(
                     local_borg_version,
                     global_arguments,
                     list_paths=source_sample_paths_subset,
-                    path_format='{xxh64} {path}{NUL}',  # noqa: FS003
+                    path_format='{xxh64} {path}{NUL}',
                     local_path=local_path,
                     remote_path=remote_path,
                 )
                 if line
-            )
+            ),
         )
 
     # Compare the source hashes with the archive hashes to see how many match.
@@ -603,20 +616,9 @@ def spot_check(
 
     if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
         raise ValueError(
-            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage'
+            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage',
         )
 
-    source_paths = collect_spot_check_source_paths(
-        repository,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-        borgmatic_runtime_directory,
-    )
-    logger.debug(f'{len(source_paths)} total source paths for spot check')
-
     archive = borgmatic.borg.repo_list.resolve_archive_name(
         repository['path'],
         'latest',
@@ -628,6 +630,25 @@ def spot_check(
     )
     logger.debug(f'Using archive {archive} for spot check')
 
+    source_paths = collect_spot_check_source_paths(
+        repository,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        borgmatic_runtime_directory,
+        bootstrap_config_paths=borgmatic.actions.config.bootstrap.load_config_paths_from_archive(
+            repository['path'],
+            archive,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        ),
+    )
+    logger.debug(f'{len(source_paths)} total source paths for spot check')
+
     archive_paths = collect_spot_check_archive_paths(
         repository,
         archive,
@@ -648,7 +669,7 @@ def spot_check(
         )
         logger.debug(f'Paths in latest archive but not source paths: {truncated_archive_paths}')
         raise ValueError(
-            'Spot check failed: There are no source paths to compare against the archive'
+            'Spot check failed: There are no source paths to compare against the archive',
         )
 
     # Calculate the percentage delta between the source paths count and the archive paths count, and
@@ -656,14 +677,14 @@ def spot_check(
     count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
 
     if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
-        rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
+        rootless_source_paths = {path.lstrip(os.path.sep) for path in source_paths}
         truncated_exclusive_source_paths = textwrap.shorten(
             ', '.join(rootless_source_paths - set(archive_paths)) or 'none',
             width=MAX_SPOT_CHECK_PATHS_LENGTH,
             placeholder=' ...',
         )
         logger.debug(
-            f'Paths in source paths but not latest archive: {truncated_exclusive_source_paths}'
+            f'Paths in source paths but not latest archive: {truncated_exclusive_source_paths}',
         )
         truncated_exclusive_archive_paths = textwrap.shorten(
             ', '.join(set(archive_paths) - rootless_source_paths) or 'none',
@@ -671,10 +692,10 @@ def spot_check(
             placeholder=' ...',
         )
         logger.debug(
-            f'Paths in latest archive but not source paths: {truncated_exclusive_archive_paths}'
+            f'Paths in latest archive but not source paths: {truncated_exclusive_archive_paths}',
         )
         raise ValueError(
-            f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
+            f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)',
         )
 
     failing_paths = compare_spot_check_hashes(
@@ -700,14 +721,14 @@ def spot_check(
             placeholder=' ...',
         )
         logger.debug(
-            f'Source paths with data not matching the latest archive: {truncated_failing_paths}'
+            f'Source paths with data not matching the latest archive: {truncated_failing_paths}',
         )
         raise ValueError(
-            f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
+            f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)',
         )
 
     logger.info(
-        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
+        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta',
     )
 
 
@@ -726,11 +747,6 @@ def run_check(
 
     Raise ValueError if the Borg repository ID cannot be determined.
     '''
-    if check_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, check_arguments.repository
-    ):
-        return
-
     logger.info('Running consistency checks')
 
     repository_id = borgmatic.borg.check.get_repository_id(
@@ -744,7 +760,10 @@ def run_check(
     upgrade_check_times(config, repository_id)
     configured_checks = parse_checks(config, check_arguments.only_checks)
     archive_filter_flags = borgmatic.borg.check.make_archive_filter_flags(
-        local_borg_version, config, configured_checks, check_arguments
+        local_borg_version,
+        config,
+        configured_checks,
+        check_arguments,
     )
     archives_check_id = make_archives_check_id(archive_filter_flags)
     checks = filter_checks_on_frequency(
@@ -794,4 +813,5 @@ def run_check(
                 remote_path,
                 borgmatic_runtime_directory,
             )
+
         write_check_time(make_check_time_path(config, repository_id, 'spot'))

borgmatic/actions/compact.py

@@ -2,8 +2,6 @@ import logging
 
 import borgmatic.borg.compact
 import borgmatic.borg.feature
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -22,11 +20,6 @@ def run_compact(
     '''
     Run the "compact" action for the given repository.
     '''
-    if compact_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, compact_arguments.repository
-    ):
-        return
-
     if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
         logger.info(f'Compacting segments{dry_run_label}')
         borgmatic.borg.compact.compact_segments(

borgmatic/actions/config/bootstrap.py

@@ -5,8 +5,6 @@ import os
 import borgmatic.borg.extract
 import borgmatic.borg.repo_list
 import borgmatic.config.paths
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -16,78 +14,81 @@ def make_bootstrap_config(bootstrap_arguments):
     Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
     '''
     return {
-        'ssh_command': bootstrap_arguments.ssh_command,
+        'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory,
+        'local_path': bootstrap_arguments.local_path,
+        'remote_path': bootstrap_arguments.remote_path,
         # In case the repo has been moved or is accessed from a different path at the point of
         # bootstrapping.
         'relocated_repo_access_is_ok': True,
+        'ssh_command': bootstrap_arguments.ssh_command,
+        'user_runtime_directory': bootstrap_arguments.user_runtime_directory,
     }
 
 
-def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
+def load_config_paths_from_archive(
+    repository_path,
+    archive_name,
+    config,
+    local_borg_version,
+    global_arguments,
+    borgmatic_runtime_directory,
+):
     '''
-    Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
-    repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
-    borgmatic source directory, destination directory, and whether to strip components), the global
-    arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
-    return the config paths from the manifest.json file in the borgmatic source directory or runtime
-    directory after extracting it from the repository archive.
+    Given a repository path, an archive name, a configuration dict, the local Borg version, the
+    global arguments as an argparse.Namespace, and the borgmatic runtime directory, return the
+    config paths from the manifest.json file in the borgmatic source directory or runtime directory
+    within the repository archive.
 
     Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
     expected configuration path data.
     '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
-        {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
-    )
-    config = make_bootstrap_config(bootstrap_arguments)
-
     # Probe for the manifest file in multiple locations, as the default location has moved to the
     # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
     # still want to support reading the manifest from previously created archives as well.
-    with borgmatic.config.paths.Runtime_directory(
-        {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
-    ) as borgmatic_runtime_directory:
-        for base_directory in (
-            'borgmatic',
-            borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-            borgmatic_source_directory,
-        ):
-            borgmatic_manifest_path = 'sh:' + os.path.join(
-                base_directory, 'bootstrap', 'manifest.json'
-            )
+    for base_directory in (
+        'borgmatic',
+        borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+        borgmatic.config.paths.get_borgmatic_source_directory(config),
+    ):
+        borgmatic_manifest_path = 'sh:' + os.path.join(
+            base_directory,
+            'bootstrap',
+            'manifest.json',
+        )
 
-            extract_process = borgmatic.borg.extract.extract_archive(
-                global_arguments.dry_run,
-                bootstrap_arguments.repository,
-                archive_name,
-                [borgmatic_manifest_path],
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path=bootstrap_arguments.local_path,
-                remote_path=bootstrap_arguments.remote_path,
-                extract_to_stdout=True,
-            )
-            manifest_json = extract_process.stdout.read()
+        extract_process = borgmatic.borg.extract.extract_archive(
+            global_arguments.dry_run,
+            repository_path,
+            archive_name,
+            [borgmatic_manifest_path],
+            config,
+            local_borg_version,
+            global_arguments,
+            local_path=config.get('local_path', 'borg'),
+            remote_path=config.get('remote_path'),
+            extract_to_stdout=True,
+        )
+        manifest_json = extract_process.stdout.read()
 
-            if manifest_json:
-                break
-        else:
-            raise ValueError(
-                'Cannot read configuration paths from archive due to missing bootstrap manifest'
-            )
+        if manifest_json:
+            break
+    else:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to missing archive or bootstrap manifest',
+        )
 
     try:
         manifest_data = json.loads(manifest_json)
     except json.JSONDecodeError as error:
         raise ValueError(
-            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
+            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}',
         )
 
     try:
         return manifest_data['config_paths']
     except KeyError:
         raise ValueError(
-            'Cannot read configuration paths from archive due to invalid bootstrap manifest'
+            'Cannot read configuration paths from archive due to invalid bootstrap manifest',
         )
 
 
@@ -108,9 +109,16 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
         local_path=bootstrap_arguments.local_path,
         remote_path=bootstrap_arguments.remote_path,
     )
-    manifest_config_paths = get_config_paths(
-        archive_name, bootstrap_arguments, global_arguments, local_borg_version
-    )
+
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        manifest_config_paths = load_config_paths_from_archive(
+            bootstrap_arguments.repository,
+            archive_name,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        )
 
     logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
 

borgmatic/actions/config/generate.py

@@ -19,30 +19,37 @@ def run_generate(generate_arguments, global_arguments):
     dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
 
     logger.answer(
-        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
+        f'Generating configuration files within: {generate_arguments.destination_path}{dry_run_label}'
+        if generate_arguments.split
+        else f'Generating a configuration file at: {generate_arguments.destination_path}{dry_run_label}'
     )
 
     borgmatic.config.generate.generate_sample_configuration(
         global_arguments.dry_run,
         generate_arguments.source_filename,
-        generate_arguments.destination_filename,
+        generate_arguments.destination_path,
         borgmatic.config.validate.schema_filename(),
         overwrite=generate_arguments.overwrite,
+        split=generate_arguments.split,
     )
 
     if generate_arguments.source_filename:
         logger.answer(
             f'''
-Merged in the contents of configuration file at: {generate_arguments.source_filename}
-To review the changes made, run:
-
-    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
+Merged the contents of the source configuration file: {generate_arguments.source_filename}'''
         )
 
+        if not generate_arguments.split:
+            logger.answer(
+                f'''To review the changes made, run:
+
+    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_path}''',
+            )
+
     logger.answer(
         '''
 This includes all available configuration options with example values, the few
 required options as indicated. Please edit the file to suit your needs.
 
-If you ever need help: https://torsion.org/borgmatic/#issues'''
+If you ever need help: https://torsion.org/borgmatic/#issues''',
     )

borgmatic/actions/config/validate.py

@@ -18,7 +18,7 @@ def run_validate(validate_arguments, configs):
     borgmatic.logger.add_custom_log_levels()
 
     if validate_arguments.show:
-        for config_path, config in configs.items():
+        for config in configs.values():
             if len(configs) > 1:
                 logger.answer('---')
 

borgmatic/actions/create.py

@@ -2,8 +2,10 @@ import logging
 
 import borgmatic.actions.json
 import borgmatic.borg.create
+import borgmatic.borg.feature
+import borgmatic.borg.rename
+import borgmatic.borg.repo_list
 import borgmatic.config.paths
-import borgmatic.config.validate
 import borgmatic.hooks.dispatch
 from borgmatic.actions import pattern
 
@@ -27,33 +29,36 @@ def run_create(
 
     If create_arguments.json is True, yield the JSON output from creating the archive.
     '''
-    if create_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, create_arguments.repository
-    ):
-        return
-
     if config.get('list_details') and config.get('progress'):
         raise ValueError(
-            'With the create action, only one of --list/--files/list_details and --progress/progress can be used.'
+            'With the create action, only one of --list/--files/list_details and --progress/progress can be used.',
         )
 
     if config.get('list_details') and create_arguments.json:
         raise ValueError(
-            'With the create action, only one of --list/--files/list_details and --json can be used.'
+            'With the create action, only one of --list/--files/list_details and --json can be used.',
         )
 
     logger.info(f'Creating archive{dry_run_label}')
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
     with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        patterns = pattern.process_patterns(
+            pattern.collect_patterns(config),
+            config,
+            working_directory,
+            borgmatic_runtime_directory,
+        )
+
+        original_patterns = list(patterns)
         borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
             'remove_data_source_dumps',
             config,
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
+            original_patterns,
             global_arguments.dry_run,
         )
-        patterns = pattern.process_patterns(pattern.collect_patterns(config), working_directory)
         active_dumps = borgmatic.hooks.dispatch.call_hooks(
             'dump_data_sources',
             config,
@@ -68,10 +73,20 @@ def run_create(
         # we could end up with duplicate paths that cause Borg to hang when it tries to read from
         # the same named pipe twice.
         patterns = pattern.process_patterns(
-            patterns, working_directory, skip_expand_paths=config_paths
+            patterns,
+            config,
+            working_directory,
+            borgmatic_runtime_directory,
+            skip_expand_paths=config_paths,
         )
         stream_processes = [process for processes in active_dumps.values() for process in processes]
 
+        # If we have stream processes, we first create an archive with .checkpoint suffix. This is
+        # to make sure we only create a real archive if all the streaming processes completed
+        # successfully (create_archive will fail if a streaming process fails, but the archive might
+        # have already been created at this point).
+        use_checkpoint = bool(stream_processes)
+
         json_output = borgmatic.borg.create.create_archive(
             global_arguments.dry_run,
             repository['path'],
@@ -80,19 +95,106 @@ def run_create(
             local_borg_version,
             global_arguments,
             borgmatic_runtime_directory,
+            archive_suffix='.checkpoint' if use_checkpoint else '',
             local_path=local_path,
             remote_path=remote_path,
             json=create_arguments.json,
+            comment=create_arguments.comment,
             stream_processes=stream_processes,
         )
 
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+        if use_checkpoint:
+            rename_checkpoint_archive(
+                repository['path'],
+                global_arguments,
+                config,
+                local_borg_version,
+                local_path,
+                remote_path,
+            )
 
+        if json_output:
+            output = borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+            if use_checkpoint:
+                # Patch archive name and ID
+                renamed_archive = borgmatic.borg.repo_list.get_latest_archive(
+                    repository['path'],
+                    config,
+                    local_borg_version,
+                    global_arguments,
+                    local_path,
+                    remote_path,
+                )
+
+                output['archive']['name'] = renamed_archive['name']
+                output['archive']['id'] = renamed_archive['id']
+
+            yield output
+
+        # Use the original patterns so as to disregard any modifications made by any data source
+        # hooks, e.g. via dump_data_sources() above.
         borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
             'remove_data_source_dumps',
             config,
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
+            original_patterns,
             global_arguments.dry_run,
         )
+
+
+def rename_checkpoint_archive(
+    repository_path,
+    global_arguments,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    '''
+    Renames the latest archive to not have a '.checkpoint' suffix.
+
+    Raises ValueError if
+    - there is not latest archive
+    - the latest archive does not have a '.checkpoint' suffix
+
+    Implementation note: We cannot reliably get the just created archive name.
+    So we resort to listing the archives and picking the last one.
+
+    A similar comment applies to retrieving the ID of the renamed archive.
+    '''
+    archive = borgmatic.borg.repo_list.get_latest_archive(
+        repository_path,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        consider_checkpoints=True,
+    )
+
+    archive_name = archive['name']
+
+    if not archive_name.endswith('.checkpoint'):
+        raise ValueError(f'Latest archive did not have a .checkpoint suffix. Got: {archive_name}')
+
+    new_archive_name = archive_name.removesuffix('.checkpoint')
+
+    logger.info(f'Renaming archive {archive_name} -> {new_archive_name}')
+
+    borgmatic.borg.rename.rename_archive(
+        repository_path,
+        (
+            archive['id']
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.ARCHIVE_SERIES, local_borg_version
+            )
+            else archive['name']
+        ),
+        new_archive_name,
+        global_arguments.dry_run,
+        config,
+        local_borg_version,
+        local_path,
+        remote_path,
+    )

borgmatic/actions/delete.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.actions.arguments
 import borgmatic.borg.delete
-import borgmatic.borg.repo_delete
 import borgmatic.borg.repo_list
 
 logger = logging.getLogger(__name__)
@@ -20,31 +19,28 @@ def run_delete(
     '''
     Run the "delete" action for the given repository and archive(s).
     '''
-    if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, delete_arguments.repository
-    ):
-        logger.answer('Deleting archives')
+    logger.answer('Deleting archives')
 
-        archive_name = (
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                delete_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            )
-            if delete_arguments.archive
-            else None
-        )
-
-        borgmatic.borg.delete.delete_archives(
-            repository,
+    archive_name = (
+        borgmatic.borg.repo_list.resolve_archive_name(
+            repository['path'],
+            delete_arguments.archive,
             config,
             local_borg_version,
-            borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
             global_arguments,
             local_path,
             remote_path,
         )
+        if delete_arguments.archive
+        else None
+    )
+
+    borgmatic.borg.delete.delete_archives(
+        repository,
+        config,
+        local_borg_version,
+        borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
+        global_arguments,
+        local_path,
+        remote_path,
+    )

borgmatic/actions/export_key.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.export_key
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,16 +17,13 @@ def run_export_key(
     '''
     Run the "key export" action for the given repository.
     '''
-    if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, export_arguments.repository
-    ):
-        logger.info('Exporting repository key')
-        borgmatic.borg.export_key.export_key(
-            repository['path'],
-            config,
-            local_borg_version,
-            export_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
+    logger.info('Exporting repository key')
+    borgmatic.borg.export_key.export_key(
+        repository['path'],
+        config,
+        local_borg_version,
+        export_arguments,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/export_tar.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.borg.export_tar
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -19,29 +18,26 @@ def run_export_tar(
     '''
     Run the "export-tar" action for the given repository.
     '''
-    if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, export_tar_arguments.repository
-    ):
-        logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
-        borgmatic.borg.export_tar.export_tar_archive(
-            global_arguments.dry_run,
+    logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
+    borgmatic.borg.export_tar.export_tar_archive(
+        global_arguments.dry_run,
+        repository['path'],
+        borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                export_tar_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            export_tar_arguments.paths,
-            export_tar_arguments.destination,
+            export_tar_arguments.archive,
             config,
             local_borg_version,
             global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            tar_filter=export_tar_arguments.tar_filter,
-            strip_components=export_tar_arguments.strip_components,
-        )
+            local_path,
+            remote_path,
+        ),
+        export_tar_arguments.paths,
+        export_tar_arguments.destination,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+        tar_filter=export_tar_arguments.tar_filter,
+        strip_components=export_tar_arguments.strip_components,
+    )

borgmatic/actions/extract.py

@@ -2,8 +2,6 @@ import logging
 
 import borgmatic.borg.extract
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -21,28 +19,25 @@ def run_extract(
     '''
     Run the "extract" action for the given repository.
     '''
-    if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, extract_arguments.repository
-    ):
-        logger.info(f'Extracting archive {extract_arguments.archive}')
-        borgmatic.borg.extract.extract_archive(
-            global_arguments.dry_run,
+    logger.info(f'Extracting archive {extract_arguments.archive}')
+    borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        repository['path'],
+        borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                extract_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            extract_arguments.paths,
+            extract_arguments.archive,
             config,
             local_borg_version,
             global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            destination_path=extract_arguments.destination,
-            strip_components=extract_arguments.strip_components,
-        )
+            local_path,
+            remote_path,
+        ),
+        extract_arguments.paths,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+        destination_path=extract_arguments.destination,
+        strip_components=extract_arguments.strip_components,
+    )

borgmatic/actions/import_key.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.import_key
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,16 +17,13 @@ def run_import_key(
     '''
     Run the "key import" action for the given repository.
     '''
-    if import_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, import_arguments.repository
-    ):
-        logger.info('Importing repository key')
-        borgmatic.borg.import_key.import_key(
-            repository['path'],
-            config,
-            local_borg_version,
-            import_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
+    logger.info('Importing repository key')
+    borgmatic.borg.import_key.import_key(
+        repository['path'],
+        config,
+        local_borg_version,
+        import_arguments,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/info.py

@@ -4,7 +4,6 @@ import borgmatic.actions.arguments
 import borgmatic.actions.json
 import borgmatic.borg.info
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -23,28 +22,26 @@ def run_info(
 
     If info_arguments.json is True, yield the JSON output from the info for the archive.
     '''
-    if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, info_arguments.repository
-    ):
-        if not info_arguments.json:
-            logger.answer('Displaying archive summary information')
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            info_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        json_output = borgmatic.borg.info.display_archives_info(
-            repository['path'],
-            config,
-            local_borg_version,
-            borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+    if not info_arguments.json:
+        logger.answer('Displaying archive summary information')
+
+    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        info_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    json_output = borgmatic.borg.info.display_archives_info(
+        repository['path'],
+        config,
+        local_borg_version,
+        borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    if json_output:
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/list.py

@@ -3,7 +3,6 @@ import logging
 import borgmatic.actions.arguments
 import borgmatic.actions.json
 import borgmatic.borg.list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -22,32 +21,29 @@ def run_list(
 
     If list_arguments.json is True, yield the JSON output from listing the archive.
     '''
-    if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, list_arguments.repository
-    ):
-        if not list_arguments.json:
-            if list_arguments.find_paths:  # pragma: no cover
-                logger.answer('Searching archives')
-            elif not list_arguments.archive:  # pragma: no cover
-                logger.answer('Listing archives')
+    if not list_arguments.json:
+        if list_arguments.find_paths:  # pragma: no cover
+            logger.answer('Searching archives')
+        elif not list_arguments.archive:  # pragma: no cover
+            logger.answer('Listing archives')
 
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
-            list_arguments.archive,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        json_output = borgmatic.borg.list.list_archive(
-            repository['path'],
-            config,
-            local_borg_version,
-            borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        list_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    json_output = borgmatic.borg.list.list_archive(
+        repository['path'],
+        config,
+        local_borg_version,
+        borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    if json_output:
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/mount.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.borg.mount
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -19,29 +18,26 @@ def run_mount(
     '''
     Run the "mount" action for the given repository.
     '''
-    if mount_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, mount_arguments.repository
-    ):
-        if mount_arguments.archive:
-            logger.info(f'Mounting archive {mount_arguments.archive}')
-        else:  # pragma: nocover
-            logger.info('Mounting repository')
+    if mount_arguments.archive:
+        logger.info(f'Mounting archive {mount_arguments.archive}')
+    else:  # pragma: nocover
+        logger.info('Mounting repository')
 
-        borgmatic.borg.mount.mount_archive(
+    borgmatic.borg.mount.mount_archive(
+        repository['path'],
+        borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                mount_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            mount_arguments,
+            mount_arguments.archive,
             config,
             local_borg_version,
             global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
+            local_path,
+            remote_path,
+        ),
+        mount_arguments,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/pattern.py

@@ -47,7 +47,8 @@ def collect_patterns(config):
         return (
             tuple(
                 borgmatic.borg.pattern.Pattern(
-                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
+                    source_directory,
+                    source=borgmatic.borg.pattern.Pattern_source.CONFIG,
                 )
                 for source_directory in config.get('source_directories', ())
             )
@@ -67,7 +68,7 @@ def collect_patterns(config):
             + tuple(
                 parse_pattern(pattern_line.strip())
                 for filename in config.get('patterns_from', ())
-                for pattern_line in open(filename).readlines()
+                for pattern_line in open(filename, encoding='utf-8').readlines()
                 if not pattern_line.lstrip().startswith('#')
                 if pattern_line.strip()
             )
@@ -77,7 +78,7 @@ def collect_patterns(config):
                     borgmatic.borg.pattern.Pattern_style.FNMATCH,
                 )
                 for filename in config.get('exclude_from', ())
-                for exclude_line in open(filename).readlines()
+                for exclude_line in open(filename, encoding='utf-8').readlines()
                 if not exclude_line.lstrip().startswith('#')
                 if exclude_line.strip()
             )
@@ -112,9 +113,8 @@ def expand_directory(directory, working_directory):
             glob_path
             # If these are equal, that means we didn't add any working directory prefix above.
             if normalized_directory == expanded_directory
-            # Remove the working directory prefix that we added above in order to make glob() work.
-            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
-            # hack.
+            # Remove the working directory prefix added above in order to make glob() work. We
+            # can't use os.path.relpath() here because it collapses any use of Borg's slashdot hack.
             else glob_path.removeprefix(working_directory_prefix)
         )
         for glob_path in glob_paths
@@ -161,7 +161,7 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None):
                 )
             )
             for pattern in patterns
-        )
+        ),
     )
 
 
@@ -180,8 +180,10 @@ def get_existent_path_or_parent(path):
     try:
         return next(
             candidate_path
-            for candidate_path in (path,)
-            + tuple(str(parent) for parent in pathlib.PurePath(path).parents)
+            for candidate_path in (
+                path,
+                *tuple(str(parent) for parent in pathlib.PurePath(path).parents),
+            )
             if os.path.exists(candidate_path)
         )
     except StopIteration:
@@ -219,49 +221,75 @@ def device_map_patterns(patterns, working_directory=None):
         for pattern in patterns
         for existent_path in (
             get_existent_path_or_parent(
-                os.path.join(working_directory or '', pattern.path.lstrip('^'))
+                os.path.join(working_directory or '', pattern.path.lstrip('^')),
             ),
         )
     )
 
 
-def deduplicate_patterns(patterns):
+def deduplicate_runtime_directory_patterns(patterns, config, borgmatic_runtime_directory=None):
     '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
-    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
-    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, the borgmatic runtime directory,
+    and a configuration dict, return them without any duplicate root child patterns that contain the
+    runtime directory. For instance, if two root patterns are given with paths "/foo" and
+    "/foo/bar", and the runtime directory is "/foo/bar", return just the "/foo" pattern. Non-root
+    patterns and patterns not containing the runtime directory are passed through without
     modification.
 
-    The one exception to deduplication is two paths are on different filesystems (devices). In that
-    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
-    one_file_system option is true).
+    One exception to deduplication is if two paths are on different filesystems (devices) and
+    "one_file_system" is True in the given configuration. In that case, the paths won't get
+    deduplicated, because Borg won't cross filesystem boundaries when "one_file_system" is True.
 
-    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
-    child patterns, because it will naturally spider the contents of the parent pattern's path. And
-    there are cases where Borg coming across the same file twice will result in duplicate reads and
-    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
-    Borg.
+    The idea is that if Borg is given a root parent pattern containing the borgmatic runtime
+    directory, then Borg doesn't also need to be given child patterns, because it will naturally
+    spider the contents of the parent pattern's path. Additionally, there are cases where Borg
+    coming across the same file twice will result in duplicate reads and even hangs, e.g. when a
+    database hook in the borgmatic runtime directory is using a named pipe for streaming database
+    dumps to Borg.
+
+    This deduplication is limited to the borgmatic runtime directory (where borgmatic's named pipes
+    exist), because there are other legitimate use cases for parent and child patterns to both exist
+    in patterns. For instance, with some snapshotted filesystems, snapshots don't traverse from a
+    parent filesystem to a child and therefore both need to remain in patterns.
+
+    And for the case of named pipes outside of the borgmatic runtime directory, there is code
+    elsewhere (in the "create" action) that auto-excludes special files to prevent Borg hangs.
     '''
+    if borgmatic_runtime_directory is None:
+        return patterns
+
     deduplicated = {}  # Use just the keys as an ordered set.
+    runtime_directory_parents = set(pathlib.PurePath(borgmatic_runtime_directory).parents).union(
+        {pathlib.PurePath(borgmatic_runtime_directory)}
+    )
 
     for pattern in patterns:
         if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
             deduplicated[pattern] = True
             continue
 
-        parents = pathlib.PurePath(pattern.path).parents
+        pattern_parents = pathlib.PurePath(pattern.path).parents
 
-        # If another directory in the given list is a parent of current directory (even n levels up)
-        # and both are on the same filesystem, then the current directory is a duplicate.
+        # If:
+        #
+        #   1. another pattern is a parent of the current pattern (even n levels up),
+        #   2. both patterns are parents of the runtime directory (even n levels up),
+        #   3. and both patterns are on the same filesystem (or one_file_system is not set)
+        #
+        # ... then consider the current pattern as a duplicate.
         for other_pattern in patterns:
             if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
                 continue
 
-            if any(
-                pathlib.PurePath(other_pattern.path) == parent
-                and pattern.device is not None
-                and other_pattern.device == pattern.device
-                for parent in parents
+            device_matches = pattern.device is not None and (
+                other_pattern.device == pattern.device or config.get('one_file_system') is not True
+            )
+
+            if (
+                pathlib.PurePath(other_pattern.path) in pattern_parents
+                and pathlib.PurePosixPath(other_pattern.path) in runtime_directory_parents
+                and pathlib.PurePosixPath(pattern.path) in runtime_directory_parents
+                and device_matches
             ):
                 break
         else:
@@ -270,23 +298,30 @@ def deduplicate_patterns(patterns):
     return tuple(deduplicated.keys())
 
 
-def process_patterns(patterns, working_directory, skip_expand_paths=None):
+def process_patterns(
+    patterns, config, working_directory, borgmatic_runtime_directory=None, skip_expand_paths=None
+):
     '''
-    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
-    "root" patterns, returning the resulting root and non-root patterns as a list.
+    Given a sequence of Borg patterns, a configuration dict, a configured working directory, the
+    borgmatic runtime directory, and a sequence of paths to skip path expansion for, expand and
+    deduplicate any "root" patterns, returning the resulting root and non-root patterns as a list.
 
-    If any paths are given to skip, don't expand them.
+    If the borgmatic runtime directory is None, then don't deduplicate patterns. Deduplication is
+    really only necessary for the "create" action when the runtime directory might contain named
+    pipes for database dumps.
     '''
     skip_paths = set(skip_expand_paths or ())
 
     return list(
-        deduplicate_patterns(
+        deduplicate_runtime_directory_patterns(
             device_map_patterns(
                 expand_patterns(
                     patterns,
                     working_directory=working_directory,
                     skip_paths=skip_paths,
-                )
-            )
-        )
+                ),
+            ),
+            config,
+            borgmatic_runtime_directory,
+        ),
     )

borgmatic/actions/prune.py

@@ -1,8 +1,6 @@
 import logging
 
 import borgmatic.borg.prune
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -21,11 +19,6 @@ def run_prune(
     '''
     Run the "prune" action for the given repository.
     '''
-    if prune_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, prune_arguments.repository
-    ):
-        return
-
     logger.info(f'Pruning archives{dry_run_label}')
     borgmatic.borg.prune.prune_archives(
         global_arguments.dry_run,

borgmatic/actions/recreate.py

@@ -4,7 +4,6 @@ import subprocess
 import borgmatic.borg.info
 import borgmatic.borg.recreate
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 from borgmatic.actions.pattern import collect_patterns, process_patterns
 
 logger = logging.getLogger(__name__)
@@ -25,60 +24,60 @@ def run_recreate(
     '''
     Run the "recreate" action for the given repository.
     '''
-    if recreate_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, recreate_arguments.repository
-    ):
-        if recreate_arguments.archive:
-            logger.answer(f'Recreating archive {recreate_arguments.archive}')
-        else:
-            logger.answer('Recreating repository')
+    if recreate_arguments.archive:
+        logger.answer(f'Recreating archive {recreate_arguments.archive}')
+    else:
+        logger.answer('Recreating repository')
 
-        # Collect and process patterns.
-        processed_patterns = process_patterns(
-            collect_patterns(config), borgmatic.config.paths.get_working_directory(config)
+    # Collect and process patterns.
+    processed_patterns = process_patterns(
+        collect_patterns(config),
+        config,
+        borgmatic.config.paths.get_working_directory(config),
+    )
+
+    archive = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        recreate_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    if archive and archive.endswith('.recreate'):
+        if recreate_arguments.archive == 'latest':
+            raise ValueError(
+                f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.',
+            )
+
+        raise ValueError(
+            f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.',
         )
 
-        archive = borgmatic.borg.repo_list.resolve_archive_name(
+    try:
+        borgmatic.borg.recreate.recreate_archive(
             repository['path'],
-            recreate_arguments.archive,
+            archive,
             config,
             local_borg_version,
+            recreate_arguments,
             global_arguments,
-            local_path,
-            remote_path,
+            local_path=local_path,
+            remote_path=remote_path,
+            patterns=processed_patterns,
         )
-
-        if archive and archive.endswith('.recreate'):
-            if recreate_arguments.archive == 'latest':
+    except subprocess.CalledProcessError as error:
+        if error.returncode == BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS:
+            if recreate_arguments.target:
                 raise ValueError(
-                    f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.'
-                )
-            else:
-                raise ValueError(
-                    f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.'
+                    f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.',
                 )
 
-        try:
-            borgmatic.borg.recreate.recreate_archive(
-                repository['path'],
-                archive,
-                config,
-                local_borg_version,
-                recreate_arguments,
-                global_arguments,
-                local_path=local_path,
-                remote_path=remote_path,
-                patterns=processed_patterns,
-            )
-        except subprocess.CalledProcessError as error:
-            if error.returncode == BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS:
-                if recreate_arguments.target:
-                    raise ValueError(
-                        f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.'
-                    )
-                elif archive:
-                    raise ValueError(
-                        f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.'
-                    )
+            if archive:
+                raise ValueError(
+                    f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.',
+                )
 
-            raise
+        raise

borgmatic/actions/repo_create.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.repo_create
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,18 +17,13 @@ def run_repo_create(
     '''
     Run the "repo-create" action for the given repository.
     '''
-    if repo_create_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, repo_create_arguments.repository
-    ):
-        return
-
     logger.info('Creating repository')
 
     encryption_mode = repo_create_arguments.encryption_mode or repository.get('encryption')
 
     if not encryption_mode:
         raise ValueError(
-            'With the repo-create action, either the --encryption flag or the repository encryption option is required.'
+            'With the repo-create action, either the --encryption flag or the repository encryption option is required.',
         )
 
     borgmatic.borg.repo_create.create_repository(

borgmatic/actions/repo_delete.py

@@ -17,19 +17,16 @@ def run_repo_delete(
     '''
     Run the "repo-delete" action for the given repository.
     '''
-    if repo_delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_delete_arguments.repository
-    ):
-        logger.answer(
-            'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '')
-        )
+    logger.answer(
+        'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else ''),
+    )
 
-        borgmatic.borg.repo_delete.delete_repository(
-            repository,
-            config,
-            local_borg_version,
-            repo_delete_arguments,
-            global_arguments,
-            local_path,
-            remote_path,
-        )
+    borgmatic.borg.repo_delete.delete_repository(
+        repository,
+        config,
+        local_borg_version,
+        repo_delete_arguments,
+        global_arguments,
+        local_path,
+        remote_path,
+    )

borgmatic/actions/repo_info.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.actions.json
 import borgmatic.borg.repo_info
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -21,20 +20,17 @@ def run_repo_info(
 
     If repo_info_arguments.json is True, yield the JSON output from the info for the repository.
     '''
-    if repo_info_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_info_arguments.repository
-    ):
-        if not repo_info_arguments.json:
-            logger.answer('Displaying repository summary information')
+    if not repo_info_arguments.json:
+        logger.answer('Displaying repository summary information')
 
-        json_output = borgmatic.borg.repo_info.display_repository_info(
-            repository['path'],
-            config,
-            local_borg_version,
-            repo_info_arguments=repo_info_arguments,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+    json_output = borgmatic.borg.repo_info.display_repository_info(
+        repository['path'],
+        config,
+        local_borg_version,
+        repo_info_arguments=repo_info_arguments,
+        global_arguments=global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )
+    if json_output:
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/repo_list.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.actions.json
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -21,20 +20,17 @@ def run_repo_list(
 
     If repo_list_arguments.json is True, yield the JSON output from listing the repository.
     '''
-    if repo_list_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_list_arguments.repository
-    ):
-        if not repo_list_arguments.json:
-            logger.answer('Listing repository')
+    if not repo_list_arguments.json:
+        logger.answer('Listing repository')
 
-        json_output = borgmatic.borg.repo_list.list_repository(
-            repository['path'],
-            config,
-            local_borg_version,
-            repo_list_arguments=repo_list_arguments,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+    json_output = borgmatic.borg.repo_list.list_repository(
+        repository['path'],
+        config,
+        local_borg_version,
+        repo_list_arguments=repo_list_arguments,
+        global_arguments=global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )
+    if json_output:
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/restore.py

@@ -5,12 +5,11 @@ import pathlib
 import shutil
 import tempfile
 
+import borgmatic.actions.pattern
 import borgmatic.borg.extract
 import borgmatic.borg.list
-import borgmatic.borg.mount
 import borgmatic.borg.repo_list
 import borgmatic.config.paths
-import borgmatic.config.validate
 import borgmatic.hooks.data_source.dump
 import borgmatic.hooks.dispatch
 
@@ -22,8 +21,8 @@ UNSPECIFIED = object()
 
 Dump = collections.namedtuple(
     'Dump',
-    ('hook_name', 'data_source_name', 'hostname', 'port'),
-    defaults=('localhost', None),
+    ('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'),
+    defaults=(None, None, None, None),
 )
 
 
@@ -33,17 +32,24 @@ def dumps_match(first, second, default_port=None):
     indicates that the field should match any value. If a default port is given, then consider any
     dump having that port to match with a dump having a None port.
     '''
-    for field_name in first._fields:
+    # label kinda counts as an unique id, if they match ignore host/container/port
+    if first.label not in {None, UNSPECIFIED} and first.label == second.label:
+        field_list = ('hook_name', 'data_source_name')
+    else:
+        field_list = Dump._fields
+
+    for field_name in field_list:
         first_value = getattr(first, field_name)
         second_value = getattr(second, field_name)
 
         if default_port is not None and field_name == 'port':
             if first_value == default_port and second_value is None:
                 continue
+
             if second_value == default_port and first_value is None:
                 continue
 
-        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
+        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:  # noqa: PLR1714
             continue
 
         if first_value != second_value:
@@ -56,16 +62,19 @@ def render_dump_metadata(dump):
     '''
     Given a Dump instance, make a display string describing it for use in log messages.
     '''
+    label = dump.label or UNSPECIFIED
     name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
-    hostname = dump.hostname or UNSPECIFIED
+    host = dump.container or dump.hostname or UNSPECIFIED
     port = None if dump.port is UNSPECIFIED else dump.port
 
-    if port:
-        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
+    if label is not UNSPECIFIED:
+        metadata = f'{name}@{label}'
+    elif port:
+        metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}'
     else:
-        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
+        metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}'
 
-    if dump.hook_name not in (None, UNSPECIFIED):
+    if dump.hook_name not in {None, UNSPECIFIED}:
         return f'{metadata} ({dump.hook_name})'
 
     return metadata
@@ -98,8 +107,10 @@ def get_configured_data_source(config, restore_dump):
             Dump(
                 hook_name,
                 hook_data_source.get('name'),
-                hook_data_source.get('hostname', 'localhost'),
+                hook_data_source.get('hostname'),
                 hook_data_source.get('port'),
+                hook_data_source.get('label') or UNSPECIFIED,
+                hook_data_source.get('container'),
             ),
             restore_dump,
             default_port,
@@ -111,14 +122,15 @@ def get_configured_data_source(config, restore_dump):
 
     if len(matching_dumps) > 1:
         raise ValueError(
-            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
+            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured',
         )
 
     return matching_dumps[0]
 
 
 def strip_path_prefix_from_extracted_dump_destination(
-    destination_path, borgmatic_runtime_directory
+    destination_path,
+    borgmatic_runtime_directory,
 ):
     '''
     Directory-format dump files get extracted into a temporary directory containing a path prefix
@@ -145,7 +157,8 @@ def strip_path_prefix_from_extracted_dump_destination(
             continue
 
         shutil.move(
-            subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)
+            subdirectory_path,
+            os.path.join(borgmatic_runtime_directory, databases_directory),
         )
         break
 
@@ -169,7 +182,14 @@ def restore_single_dump(
     that data source from the archive.
     '''
     dump_metadata = render_dump_metadata(
-        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
+        Dump(
+            hook_name,
+            data_source['name'],
+            data_source.get('hostname'),
+            data_source.get('port'),
+            data_source.get('label') or UNSPECIFIED,
+            data_source.get('container'),
+        ),
     )
 
     logger.info(f'Restoring data source {dump_metadata}')
@@ -197,8 +217,8 @@ def restore_single_dump(
             archive=archive_name,
             paths=[
                 borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
-                    dump_patterns
-                )
+                    dump_patterns,
+                ),
             ],
             config=config,
             local_borg_version=local_borg_version,
@@ -213,7 +233,8 @@ def restore_single_dump(
 
         if destination_path and not global_arguments.dry_run:
             strip_path_prefix_from_extracted_dump_destination(
-                destination_path, borgmatic_runtime_directory
+                destination_path,
+                borgmatic_runtime_directory,
             )
     finally:
         if destination_path and not global_arguments.dry_run:
@@ -248,8 +269,68 @@ def collect_dumps_from_archive(
     borgmatic runtime directory, query the archive for the names of data sources dumps it contains
     and return them as a set of Dump instances.
     '''
+    dumps_from_archive = set()
+
+    # There is (at most) one dump metadata file per data source hook. Load each.
+    for dumps_metadata_path in borgmatic.borg.list.capture_archive_listing(
+        repository,
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        list_paths=[
+            'sh:'
+            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
+                base_directory,
+                '*_databases/dumps.json',
+            )
+            # Probe for dump metadata files in multiple locations, as the default location is
+            # "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the
+            # borgmatic runtime directory for older versions of Borg.
+            for base_directory in (
+                'borgmatic',
+                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+            )
+        ],
+        local_path=local_path,
+        remote_path=remote_path,
+    ):
+        if not dumps_metadata_path:
+            continue
+
+        dumps_from_archive.update(
+            set(
+                borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata(
+                    borgmatic.borg.extract.extract_archive(
+                        global_arguments.dry_run,
+                        repository,
+                        archive,
+                        [dumps_metadata_path],
+                        config,
+                        local_borg_version,
+                        global_arguments,
+                        local_path=local_path,
+                        remote_path=remote_path,
+                        extract_to_stdout=True,
+                    )
+                    .stdout.read()
+                    .decode(),
+                    dumps_metadata_path,
+                )
+            )
+        )
+
+    # If we've successfully loaded any dumps metadata, we're done.
+    if dumps_from_archive:
+        logger.debug('Collecting database dumps from archive data source dumps metadata files')
+
+        return dumps_from_archive
+
+    # No dumps metadata files were found, so for backwards compatibility, fall back to parsing the
+    # paths of dumps found in the archive to get their respective dump metadata.
+    logger.debug('Collecting database dumps from archive data source dump paths (fallback)')
     borgmatic_source_directory = str(
-        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
+        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)),
     )
 
     # Probe for the data source dumps in multiple locations, as the default location has moved to
@@ -264,7 +345,8 @@ def collect_dumps_from_archive(
         list_paths=[
             'sh:'
             + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
-                base_directory, '*_databases/*/*'
+                base_directory,
+                '*_databases/*/*',
             )
             for base_directory in (
                 'borgmatic',
@@ -276,9 +358,6 @@ def collect_dumps_from_archive(
         remote_path=remote_path,
     )
 
-    # Parse the paths of dumps found in the archive to get their respective dump metadata.
-    dumps_from_archive = set()
-
     for dump_path in dump_paths:
         if not dump_path:
             continue
@@ -291,7 +370,8 @@ def collect_dumps_from_archive(
         ):
             try:
                 (hook_name, host_and_port, data_source_name) = dump_path.split(
-                    base_directory + os.path.sep, 1
+                    base_directory + os.path.sep,
+                    1,
                 )[1].split(os.path.sep)[0:3]
             except (ValueError, IndexError):
                 continue
@@ -308,13 +388,17 @@ def collect_dumps_from_archive(
             except (ValueError, TypeError):
                 port = None
 
-            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
+            dumps_from_archive.add(
+                Dump(
+                    hook_name, data_source_name, None if hostname == 'localhost' else hostname, port
+                )
+            )
 
             # We've successfully parsed the dump path, so need to probe any further.
             break
         else:
             logger.warning(
-                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
+                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}',
             )
 
     return dumps_from_archive
@@ -345,6 +429,8 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
                 data_source_name=name,
                 hostname=restore_arguments.original_hostname or UNSPECIFIED,
                 port=restore_arguments.original_port,
+                label=restore_arguments.original_label or UNSPECIFIED,
+                container=restore_arguments.original_container or UNSPECIFIED,
             )
             for name in restore_arguments.data_sources or (UNSPECIFIED,)
         }
@@ -352,13 +438,17 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
         or restore_arguments.data_sources
         or restore_arguments.original_hostname
         or restore_arguments.original_port
+        or restore_arguments.original_label
+        or restore_arguments.original_container
         else {
             Dump(
                 hook_name=UNSPECIFIED,
                 data_source_name='all',
                 hostname=UNSPECIFIED,
                 port=UNSPECIFIED,
-            )
+                label=UNSPECIFIED,
+                container=UNSPECIFIED,
+            ),
         }
     )
     missing_dumps = set()
@@ -385,7 +475,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
             dumps_to_restore.add(matching_dumps[0])
         else:
             raise ValueError(
-                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
+                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.',
             )
 
     if missing_dumps:
@@ -394,7 +484,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
         )
 
         raise ValueError(
-            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
+            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive",
         )
 
     return dumps_to_restore
@@ -410,14 +500,15 @@ def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
         raise ValueError('No data source dumps were found to restore')
 
     missing_dumps = sorted(
-        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
+        dumps_to_restore - dumps_actually_restored,
+        key=lambda dump: dump.data_source_name,
     )
 
     if missing_dumps:
         rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
 
         raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
+            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration",
         )
 
 
@@ -431,25 +522,26 @@ def run_restore(
     remote_path,
 ):
     '''
-    Run the "restore" action for the given repository, but only if the repository matches the
-    requested repository in restore arguments.
+    Run the "restore" action for the given repository.
 
     Raise ValueError if a configured data source could not be found to restore or there's no
     matching dump in the archive.
     '''
-    if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, restore_arguments.repository
-    ):
-        return
-
     logger.info(f'Restoring data sources from archive {restore_arguments.archive}')
+    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        patterns = borgmatic.actions.pattern.process_patterns(
+            borgmatic.actions.pattern.collect_patterns(config),
+            config,
+            working_directory,
+        )
         borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
             'remove_data_source_dumps',
             config,
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
+            patterns,
             global_arguments.dry_run,
         )
 
@@ -476,6 +568,7 @@ def run_restore(
 
         dumps_actually_restored = set()
         connection_params = {
+            'container': restore_arguments.container,
             'hostname': restore_arguments.hostname,
             'port': restore_arguments.port,
             'username': restore_arguments.username,
@@ -489,13 +582,19 @@ def run_restore(
                 config,
                 restore_dump,
             )
-
             # For a dump that wasn't found via an exact match in the configuration, try to fallback
             # to an "all" data source.
             if not found_data_source:
                 found_data_source = get_configured_data_source(
                     config,
-                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
+                    Dump(
+                        restore_dump.hook_name,
+                        'all',
+                        restore_dump.hostname,
+                        restore_dump.port,
+                        restore_dump.label,
+                        restore_dump.container,
+                    ),
                 )
 
                 if not found_data_source:
@@ -515,7 +614,7 @@ def run_restore(
                 remote_path,
                 archive_name,
                 restore_dump.hook_name,
-                dict(found_data_source, **{'schemas': restore_arguments.schemas}),
+                dict(found_data_source, schemas=restore_arguments.schemas),
                 connection_params,
                 borgmatic_runtime_directory,
             )
@@ -525,6 +624,7 @@ def run_restore(
             config,
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
+            patterns,
             global_arguments.dry_run,
         )
 

borgmatic/actions/transfer.py

@@ -19,7 +19,7 @@ def run_transfer(
     '''
     if transfer_arguments.archive and config.get('match_archives'):
         raise ValueError(
-            'With the transfer action, only one of --archive and --match-archives/match_archives can be used.'
+            'With the transfer action, only one of --archive and --match-archives/match_archives can be used.',
         )
 
     logger.info('Transferring archives to repository')

borgmatic/borg/borg.py

@@ -39,9 +39,9 @@ def run_arbitrary_borg(
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
 
-        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
+        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES:
             logger.warning(
-                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
+                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}",
             )
     except IndexError:
         borg_command = ()
@@ -57,16 +57,14 @@ def run_arbitrary_borg(
         + command_options
     )
 
-    return execute_command(
+    return execute_command(  # noqa: S604
         tuple(shlex.quote(part) for part in full_command),
         output_file=DO_NOT_CAPTURE,
         shell=True,
         environment=dict(
             (environment.make_environment(config) or {}),
-            **{
-                'BORG_REPO': repository_path,
-                'ARCHIVE': archive if archive else '',
-            },
+            BORG_REPO=repository_path,
+            ARCHIVE=archive if archive else '',
         ),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,

borgmatic/borg/break_lock.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, flags
@@ -22,6 +23,7 @@ def break_lock(
     '''
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('break_lock', '')
 
     full_command = (
         (local_path, 'break-lock')
@@ -31,6 +33,7 @@ def break_lock(
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 

borgmatic/borg/change_passphrase.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.execute
@@ -25,6 +26,7 @@ def change_passphrase(
     borgmatic.logger.add_custom_log_levels()
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('key_change_passphrase', '')
 
     full_command = (
         (local_path, 'key', 'change-passphrase')
@@ -34,6 +36,7 @@ def change_passphrase(
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(
             repository_path,
             local_borg_version,
@@ -49,7 +52,7 @@ def change_passphrase(
     config_without_passphrase = {
         option_name: value
         for (option_name, value) in config.items()
-        if option_name not in ('encryption_passphrase', 'encryption_passcommand')
+        if option_name not in {'encryption_passphrase', 'encryption_passcommand'}
     }
 
     borgmatic.execute.execute_command(
@@ -63,5 +66,5 @@ def change_passphrase(
     )
 
     logger.answer(
-        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)"
+        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)",
     )

borgmatic/borg/check.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags, repo_info
@@ -41,11 +42,12 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument
 
     if check_last:
         logger.warning(
-            'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
+            'Ignoring check_last option, as "archives" or "data" are not in consistency checks',
         )
+
     if prefix:
         logger.warning(
-            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
+            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks',
         )
 
     return ()
@@ -75,13 +77,18 @@ def make_check_name_flags(checks, archive_filter_flags):
         return common_flags
 
     return (
-        tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives'))
+        tuple(f'--{check}-only' for check in checks if check in {'repository', 'archives'})
         + common_flags
     )
 
 
 def get_repository_id(
-    repository_path, config, local_borg_version, global_arguments, local_path, remote_path
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Given a local or remote repository path, a configuration dict, the local Borg version, global
@@ -99,7 +106,7 @@ def get_repository_id(
                 global_arguments,
                 local_path,
                 remote_path,
-            )
+            ),
         )['repository']['id']
     except (json.JSONDecodeError, KeyError):
         raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
@@ -126,8 +133,10 @@ def check_archives(
     extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
 
     verbosity_flags = ()
+
     if logger.isEnabledFor(logging.INFO):
         verbosity_flags = ('--info',)
+
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
 
@@ -171,7 +180,7 @@ def check_archives(
             + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
             + (('--progress',) if config.get('progress') else ())
-            + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+            + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
             + flags.make_repository_flags(repository_path, local_borg_version)
         )
 

borgmatic/borg/compact.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags
@@ -42,12 +43,12 @@ def compact_segments(
             if dry_run and feature.available(feature.Feature.DRY_RUN_COMPACT, local_borg_version)
             else ()
         )
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if dry_run and not feature.available(feature.Feature.DRY_RUN_COMPACT, local_borg_version):
-        logging.info('Skipping compact (dry run)')
+        logger.info('Skipping compact (dry run)')
         return
 
     execute_command(

borgmatic/borg/create.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import pathlib
+import shlex
 import stat
 import textwrap
 
@@ -44,35 +45,42 @@ def any_parent_directories(path, candidate_parents):
     return False
 
 
-def collect_special_file_paths(
+def validate_planned_backup_paths(
     dry_run,
     create_command,
     config,
+    patterns,
     local_path,
     working_directory,
     borgmatic_runtime_directory,
 ):
     '''
     Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
-    a working directory, and the borgmatic runtime directory, collect the paths for any special
-    files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
-    during a create. These are all paths that could cause Borg to hang if its --read-special flag is
-    used.
+    a working directory, and the borgmatic runtime directory, perform a "borg create --dry-run" to
+    determine whether Borg's planned paths to include in a backup look good. Specifically, if the
+    given runtime directory exists, validate that it will be included in a backup and hasn't been
+    excluded.
 
-    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
-    its own special files there for database dumps and we don't want those omitted.
-
-    Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
-    plans to backup, that means the user must have excluded the runtime directory (e.g. via
-    "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
-    consume any database dumps and therefore borgmatic will hang when it tries to do so.
+    Raise ValueError if the runtime directory has been excluded via "exclude_patterns" or similar,
+    because any features that rely on the runtime directory getting backed up will break. For
+    instance, without the runtime directory, Borg can't consume any database dumps and borgmatic may
+    hang waiting for them to be consumed.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
     # files including any named pipe we've created. And omit "--filter" because that can break the
     # paths output parsing below such that path lines no longer start with the expected "- ".
     paths_output = execute_command_and_capture_output(
-        flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
-        + ('--dry-run', '--list'),
+        (
+            *flags.omit_flag_and_value(
+                flags.omit_flag(
+                    flags.omit_flag(create_command, '--exclude-nodump'),
+                    '--log-json',
+                ),
+                '--filter',
+            ),
+            '--dry-run',
+            '--list',
+        ),
         capture_stderr=True,
         working_directory=working_directory,
         environment=environment.make_environment(config),
@@ -85,28 +93,31 @@ def collect_special_file_paths(
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.split('\n')
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
+        if path_line and path_line.startswith(('- ', '+ '))
     )
 
-    # These are the subset of those files that contain the borgmatic runtime directory.
-    paths_containing_runtime_directory = {}
+    runtime_directory_root_patterns = tuple(
+        pattern
+        for pattern in patterns
+        if any_parent_directories(pattern.path, (borgmatic_runtime_directory,))
+        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+    )
 
-    if os.path.exists(borgmatic_runtime_directory):
-        paths_containing_runtime_directory = {
-            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
-        }
-
-        # If no paths to backup contain the runtime directory, it must've been excluded.
-        if not paths_containing_runtime_directory and not dry_run:
-            raise ValueError(
-                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
-            )
+    if not dry_run and os.path.exists(borgmatic_runtime_directory):
+        # If there are any root patterns in the runtime directory that are missing from the paths
+        # Borg is planning to backup, then they must've gotten excluded, e.g. by user-configured
+        # excludes. Error accordingly.
+        for pattern in runtime_directory_root_patterns:
+            if not any(any_parent_directories(path, (pattern.path,)) for path in paths):
+                raise ValueError(
+                    f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.',
+                )
 
+    # Return the subset of output paths *not* contained within the borgmatic runtime directory. The
+    # intent is that any downstream checks using these paths should skip runtime paths that
+    # borgmatic uses for its own bookkeeping, instead focusing on user-configured paths.
     return tuple(
-        path
-        for path in paths
-        if special_file(path, working_directory)
-        if path not in paths_containing_runtime_directory
+        path for path in paths if not any_parent_directories(path, (borgmatic_runtime_directory,))
     )
 
 
@@ -121,23 +132,28 @@ def make_base_create_command(
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
+    archive_suffix='',
     local_path='borg',
     remote_path=None,
     json=False,
+    comment=None,
     stream_processes=None,
 ):
     '''
     Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version,
-    global arguments as an argparse.Namespace instance, and a sequence of borgmatic source
-    directories, return a tuple of (base Borg create command flags, Borg create command positional
-    arguments, open pattern file handle).
+    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version, global
+    arguments as an argparse.Namespace instance, the borgmatic runtime directory, a string suffix to
+    add to the archive name, the local Borg path, the remote Borg path, whether to output JSON,
+    comment text to add to the created archive, and a sequence of processes streaming data to Borg,
+    return a tuple of (base Borg create command flags, Borg create command positional arguments,
+    open pattern file handle).
     '''
     if config.get('source_directories_must_exist', False):
         borgmatic.borg.pattern.check_all_root_patterns_exist(patterns)
 
     patterns_file = borgmatic.borg.pattern.write_patterns_file(
-        patterns, borgmatic_runtime_directory
+        patterns,
+        borgmatic_runtime_directory,
     )
     checkpoint_interval = config.get('checkpoint_interval', None)
     checkpoint_volume = config.get('checkpoint_volume', None)
@@ -149,8 +165,9 @@ def make_base_create_command(
     lock_wait = config.get('lock_wait', None)
     list_filter_flags = flags.make_list_filter_flags(local_borg_version, dry_run)
     files_cache = config.get('files_cache')
-    archive_name_format = config.get(
-        'archive_name_format', flags.get_default_archive_name_format(local_borg_version)
+    archive_name_format = (
+        config.get('archive_name_format', flags.get_default_archive_name_format(local_borg_version))
+        + archive_suffix
     )
     extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
 
@@ -183,6 +200,7 @@ def make_base_create_command(
         + ('create',)
         + (('--patterns-from', patterns_file.name) if patterns_file else ())
         + flags.make_exclude_flags(config)
+        + (('--comment', comment) if comment else ())
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
@@ -207,29 +225,36 @@ def make_base_create_command(
             else ()
         )
         + (('--dry-run',) if dry_run else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
     )
 
     create_positional_arguments = flags.make_repository_archive_flags(
-        repository_path, archive_name_format, local_borg_version
+        repository_path,
+        archive_name_format,
+        local_borg_version,
+    )
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    logger.debug('Checking file paths Borg plans to include')
+    planned_backup_paths = validate_planned_backup_paths(
+        dry_run,
+        create_flags + create_positional_arguments,
+        config,
+        patterns,
+        local_path,
+        working_directory,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
     )
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
     if stream_processes and not config.get('read_special'):
         logger.warning(
-            'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
+            'Ignoring configured "read_special" value of false, as true is needed for database hooks.',
         )
-        working_directory = borgmatic.config.paths.get_working_directory(config)
 
-        logger.debug('Collecting special file paths')
-        special_file_paths = collect_special_file_paths(
-            dry_run,
-            create_flags + create_positional_arguments,
-            config,
-            local_path,
-            working_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
+        special_file_paths = tuple(
+            path for path in planned_backup_paths if special_file(path, working_directory)
         )
 
         if special_file_paths:
@@ -239,7 +264,7 @@ def make_base_create_command(
                 placeholder=' ...',
             )
             logger.warning(
-                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
+                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
             )
             patterns_file = borgmatic.borg.pattern.write_patterns_file(
                 tuple(
@@ -269,15 +294,20 @@ def create_archive(
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
+    archive_suffix='',
     local_path='borg',
     remote_path=None,
     json=False,
+    comment=None,
     stream_processes=None,
 ):
     '''
     Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of loaded configuration paths, the local Borg version, and global arguments as an
-    argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
+    sequence of loaded configuration paths, the local Borg version, global arguments as an
+    argparse.Namespace instance, the borgmatic runtime directory, a string suffix to add to the
+    archive name, the local Borg path, the remote Borg path, whether to output JSON, and comment
+    text to add to the created archive, and a sequence of processes streaming data to Borg, create a
+    Borg archive and return Borg's JSON output (if any).
 
     If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
     create command while also triggering the given processes to produce output.
@@ -286,7 +316,7 @@ def create_archive(
 
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
+    (create_flags, create_positional_arguments, _) = make_base_create_command(
         dry_run,
         repository_path,
         config,
@@ -294,9 +324,11 @@ def create_archive(
         local_borg_version,
         global_arguments,
         borgmatic_runtime_directory,
+        archive_suffix,
         local_path,
         remote_path,
         json,
+        comment,
         stream_processes,
     )
 
@@ -331,7 +363,8 @@ def create_archive(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
-    elif output_log_level is None:
+
+    if output_log_level is None:
         return execute_command_and_capture_output(
             create_flags + create_positional_arguments,
             working_directory=working_directory,
@@ -339,13 +372,15 @@ def create_archive(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
-    else:
-        execute_command(
-            create_flags + create_positional_arguments,
-            output_log_level,
-            output_file,
-            working_directory=working_directory,
-            environment=environment.make_environment(config),
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
-        )
+
+    execute_command(
+        create_flags + create_positional_arguments,
+        output_log_level,
+        output_file,
+        working_directory=working_directory,
+        environment=environment.make_environment(config),
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )
+
+    return None

borgmatic/borg/delete.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import shlex
 
 import borgmatic.borg.environment
 import borgmatic.borg.feature
@@ -11,6 +12,9 @@ import borgmatic.execute
 logger = logging.getLogger(__name__)
 
 
+FORCE_HARDER_FLAG_COUNT = 2
+
+
 def make_delete_command(
     repository,
     config,
@@ -25,6 +29,8 @@ def make_delete_command(
     arguments to the delete action as an argparse.Namespace, and global arguments, return a command
     as a tuple to delete archives from the repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('delete', '')
+
     return (
         (local_path, 'delete')
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
@@ -36,7 +42,10 @@ def make_delete_command(
         + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
         + borgmatic.borg.flags.make_flags('list', config.get('list_details'))
         + (
-            (('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
+            (
+                ('--force',)
+                + (('--force',) if delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
+            )
             if delete_arguments.force
             else ()
         )
@@ -60,6 +69,7 @@ def make_delete_command(
                 'repository',
             ),
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
     )
 
@@ -98,10 +108,11 @@ def delete_archives(
         for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
     ):
         if borgmatic.borg.feature.available(
-            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+            borgmatic.borg.feature.Feature.REPO_DELETE,
+            local_borg_version,
         ):
             logger.warning(
-                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.'
+                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.',
             )
 
         repo_delete_arguments = argparse.Namespace(

borgmatic/borg/environment.py

@@ -10,20 +10,23 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
+    'borg_key_file': 'BORG_KEY_FILE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
 }
 
-DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = {
+DEFAULT_BOOL_OPTION_TO_UNCONDITIONAL_ENVIRONMENT_VARIABLE = {
+    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
+}
+
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'debug_passphrase': 'BORG_DEBUG_PASSPHRASE',
+    'display_passphrase': 'BORG_DISPLAY_PASSPHRASE',
     'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
     'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
     'use_chunks_archive': 'BORG_USE_CHUNKS_ARCHIVE',
 }
 
-DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
-    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
-}
-
 
 def make_environment(config):
     '''
@@ -62,7 +65,8 @@ def make_environment(config):
         environment.pop('BORG_PASSCOMMAND', None)
 
     passphrase = borgmatic.hooks.credential.parse.resolve_credential(
-        config.get('encryption_passphrase'), config
+        config.get('encryption_passphrase'),
+        config,
     )
 
     if passphrase is None:
@@ -83,15 +87,15 @@ def make_environment(config):
     for (
         option_name,
         environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
         if os.environ.get(environment_variable_name) is None:
             value = config.get(option_name)
-            environment[environment_variable_name] = 'yes' if value else 'no'
+            environment[environment_variable_name] = 'YES' if value else 'NO'
 
     for (
         option_name,
         environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items():
+    ) in DEFAULT_BOOL_OPTION_TO_UNCONDITIONAL_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
         if value is not None:
             environment[environment_variable_name] = 'YES' if value else 'NO'

borgmatic/borg/export_key.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -31,11 +32,12 @@ def export_key(
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
     working_directory = borgmatic.config.paths.get_working_directory(config)
+    extra_borg_options = config.get('extra_borg_options', {}).get('key_export', '')
 
     if export_arguments.path and export_arguments.path != '-':
         if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
             raise FileExistsError(
-                f'Destination path {export_arguments.path} already exists. Aborting.'
+                f'Destination path {export_arguments.path} already exists. Aborting.',
             )
 
         output_file = None
@@ -52,6 +54,7 @@ def export_key(
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('paper', export_arguments.paper)
         + flags.make_flags('qr-html', export_arguments.qr_html)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(
             repository_path,
             local_borg_version,

borgmatic/borg/export_tar.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -34,6 +35,7 @@ def export_tar_archive(
     borgmatic.logger.add_custom_log_levels()
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('export_tar', '')
 
     full_command = (
         (local_path, 'export-tar')
@@ -47,6 +49,7 @@ def export_tar_archive(
         + (('--dry-run',) if dry_run else ())
         + (('--tar-filter', tar_filter) if tar_filter else ())
         + (('--strip-components', str(strip_components)) if strip_components else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_archive_flags(
             repository_path,
             archive,
@@ -56,13 +59,10 @@ def export_tar_archive(
         + (tuple(paths) if paths else ())
     )
 
-    if config.get('list_details'):
-        output_log_level = logging.ANSWER
-    else:
-        output_log_level = logging.INFO
+    output_log_level = logging.ANSWER if config.get('list_details') else logging.INFO
 
     if dry_run:
-        logging.info('Skipping export to tar file (dry run)')
+        logger.info('Skipping export to tar file (dry run)')
         return
 
     execute_command(

borgmatic/borg/extract.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import shlex
 import subprocess
 
 import borgmatic.config.paths
@@ -23,6 +24,7 @@ def extract_last_archive_dry_run(
     Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
     dry-run.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('extract', '')
     verbosity_flags = ()
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
@@ -51,8 +53,11 @@ def extract_last_archive_dry_run(
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + verbosity_flags
         + list_flag
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_archive_flags(
-            repository_path, last_archive_name, local_borg_version
+            repository_path,
+            last_archive_name,
+            local_borg_version,
         )
     )
 
@@ -90,6 +95,7 @@ def extract_archive(
     '''
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('extract', '')
 
     if config.get('progress') and extract_to_stdout:
         raise ValueError('progress and extract to stdout cannot both be set')
@@ -129,6 +135,7 @@ def extract_archive(
         + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if config.get('progress') else ())
         + (('--stdout',) if extract_to_stdout else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_archive_flags(
             # Make the repository path absolute so the destination directory used below via changing
             # the working directory doesn't prevent Borg from finding the repo. But also apply the
@@ -178,3 +185,5 @@ def extract_archive(
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
+
+    return None

borgmatic/borg/feature.py

@@ -36,7 +36,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
     Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
     Feature.ARCHIVE_SERIES: parse('2.0.0b11'),  # identically named archives form a series
     Feature.NO_PRUNE_STATS: parse('2.0.0b10'),  # prune --stats is not available
-    Feature.DRY_RUN_COMPACT: parse('1.4.1'),  # borg compact --dry-run support
+    Feature.DRY_RUN_COMPACT: parse('1.2.9'),  # borg compact --dry-run support
 }
 
 

borgmatic/borg/flags.py

@@ -34,7 +34,7 @@ def make_flags_from_arguments(arguments, excludes=()):
             make_flags(name, value=getattr(arguments, name))
             for name in sorted(vars(arguments))
             if name not in excludes and not name.startswith('_')
-        )
+        ),
     )
 
 
@@ -50,7 +50,7 @@ def make_repository_flags(repository_path, local_borg_version):
     ) + (repository_path,)
 
 
-ARCHIVE_HASH_PATTERN = re.compile('[0-9a-fA-F]{8,}$')
+ARCHIVE_HASH_PATTERN = re.compile(r'[0-9a-fA-F]{8,}$')
 
 
 def make_repository_archive_flags(repository_path, archive, local_borg_version):
@@ -76,8 +76,8 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
     )
 
 
-DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
-DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'  # noqa: FS003
+DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'
 
 
 def get_default_archive_name_format(local_borg_version):
@@ -90,7 +90,7 @@ def get_default_archive_name_format(local_borg_version):
     return DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES
 
 
-def make_match_archives_flags(
+def make_match_archives_flags(  # noqa: PLR0911
     match_archives,
     archive_name_format,
     local_borg_version,
@@ -115,8 +115,8 @@ def make_match_archives_flags(
                 return ('--match-archives', f'aid:{match_archives}')
 
             return ('--match-archives', match_archives)
-        else:
-            return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
+
+        return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
 
     derived_match_archives = re.sub(
         r'\{(now|utcnow|pid)([:%\w\.-]*)\}',
@@ -131,8 +131,8 @@ def make_match_archives_flags(
 
     if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
         return ('--match-archives', f'sh:{derived_match_archives}')
-    else:
-        return ('--glob-archives', f'{derived_match_archives}')
+
+    return ('--glob-archives', f'{derived_match_archives}')
 
 
 def warn_for_aggressive_archive_flags(json_command, json_output):
@@ -150,7 +150,7 @@ def warn_for_aggressive_archive_flags(json_command, json_output):
         if len(json.loads(json_output)['archives']) == 0:
             logger.warning('An archive filter was applied, but no matching archives were found.')
             logger.warning(
-                'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.'
+                'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.',
             )
     except json.JSONDecodeError as error:
         logger.debug(f'Cannot parse JSON output from archive command: {error}')
@@ -193,8 +193,8 @@ def omit_flag_and_value(arguments, flag):
     # its value.
     return tuple(
         argument
-        for (previous_argument, argument) in zip((None,) + arguments, arguments)
-        if flag not in (previous_argument, argument)
+        for (previous_argument, argument) in zip((None, *arguments), arguments)
+        if flag not in {previous_argument, argument}
         if not argument.startswith(f'{flag}=')
     )
 
@@ -209,7 +209,7 @@ def make_exclude_flags(config):
         itertools.chain.from_iterable(
             ('--exclude-if-present', if_present)
             for if_present in config.get('exclude_if_present', ())
-        )
+        ),
     )
     keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
     exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
@@ -229,10 +229,10 @@ def make_list_filter_flags(local_borg_version, dry_run):
     if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
         if show_excludes or dry_run:
             return f'{base_flags}+-'
-        else:
-            return base_flags
+
+        return base_flags
 
     if show_excludes:
         return f'{base_flags}x-'
-    else:
-        return f'{base_flags}-'
+
+    return f'{base_flags}-'

borgmatic/borg/import_key.py

@@ -1,8 +1,8 @@
 import logging
 import os
+import shlex
 
 import borgmatic.config.paths
-import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
@@ -30,6 +30,7 @@ def import_key(
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
     working_directory = borgmatic.config.paths.get_working_directory(config)
+    extra_borg_options = config.get('extra_borg_options', {}).get('key_import', '')
 
     if import_arguments.path and import_arguments.path != '-':
         if not os.path.exists(os.path.join(working_directory or '', import_arguments.path)):
@@ -48,6 +49,7 @@ def import_key(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('paper', import_arguments.paper)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(
             repository_path,
             local_borg_version,

borgmatic/borg/info.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -23,6 +24,8 @@ def make_info_command(
     arguments to the info action as an argparse.Namespace, and global arguments, return a command
     as a tuple to display summary information for archives in the repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('info', '')
+
     return (
         (local_path, 'info')
         + (
@@ -55,8 +58,10 @@ def make_info_command(
             )
         )
         + flags.make_flags_from_arguments(
-            info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
+            info_arguments,
+            excludes=('repository', 'archive', 'prefix', 'match_archives'),
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
@@ -119,3 +124,5 @@ def display_archives_info(
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
+
+    return None

borgmatic/borg/list.py

@@ -2,6 +2,7 @@ import argparse
 import copy
 import logging
 import re
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -17,7 +18,9 @@ MAKE_FLAGS_EXCLUDES = (
     'archive',
     'paths',
     'find_paths',
-) + ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST
+    'format',
+    *ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST,
+)
 
 
 def make_list_command(
@@ -34,6 +37,8 @@ def make_list_command(
     and local and remote Borg paths, return a command as a tuple to list archives or paths within an
     archive.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('list', '')
+
     return (
         (local_path, 'list')
         + (
@@ -50,10 +55,14 @@ def make_list_command(
         + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', config.get('log_json'))
         + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + flags.make_flags('format', list_arguments.format or config.get('file_list_format'))
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (
             flags.make_repository_archive_flags(
-                repository_path, list_arguments.archive, local_borg_version
+                repository_path,
+                list_arguments.archive,
+                local_borg_version,
             )
             if list_arguments.archive
             else flags.make_repository_flags(repository_path, local_borg_version)
@@ -115,10 +124,10 @@ def capture_archive_listing(
                 argparse.Namespace(
                     repository=repository_path,
                     archive=archive,
-                    paths=[path for path in list_paths] if list_paths else None,
+                    paths=list(list_paths) if list_paths else None,
                     find_paths=None,
                     json=None,
-                    format=path_format or '{path}{NUL}',  # noqa: FS003
+                    format=path_format or '{path}{NUL}',
                 ),
                 global_arguments,
                 local_path,
@@ -130,7 +139,7 @@ def capture_archive_listing(
             borg_exit_codes=config.get('borg_exit_codes'),
         )
         .strip('\0')
-        .split('\0')
+        .split('\0'),
     )
 
 
@@ -156,7 +165,7 @@ def list_archive(
     if not list_arguments.archive and not list_arguments.find_paths:
         if feature.available(feature.Feature.REPO_LIST, local_borg_version):
             logger.warning(
-                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.'
+                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.',
             )
 
         repo_list_arguments = argparse.Namespace(
@@ -184,12 +193,12 @@ def list_archive(
         for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
             if getattr(list_arguments, name, None):
                 logger.warning(
-                    f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag."
+                    f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag.",
                 )
 
     if list_arguments.json:
         raise ValueError(
-            'The --json flag on the list action is not supported when using the --archive/--find flags.'
+            'The --json flag on the list action is not supported when using the --archive/--find flags.',
         )
 
     borg_exit_codes = config.get('borg_exit_codes')
@@ -227,7 +236,7 @@ def list_archive(
                 borg_exit_codes=borg_exit_codes,
             )
             .strip('\n')
-            .splitlines()
+            .splitlines(),
         )
     else:
         archive_lines = (list_arguments.archive,)
@@ -262,3 +271,5 @@ def list_archive(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
+
+    return None

borgmatic/borg/mount.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags
@@ -25,6 +26,7 @@ def mount_archive(
     '''
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('mount', '')
 
     full_command = (
         (local_path, 'mount')
@@ -39,6 +41,7 @@ def mount_archive(
             excludes=('repository', 'archive', 'mount_point', 'paths', 'options'),
         )
         + (('-o', mount_arguments.options) if mount_arguments.options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (
             (
                 flags.make_repository_flags(repository_path, local_borg_version)

borgmatic/borg/pattern.py

@@ -4,8 +4,6 @@ import logging
 import os
 import tempfile
 
-import borgmatic.borg.pattern
-
 logger = logging.getLogger(__name__)
 
 
@@ -59,9 +57,9 @@ Pattern = collections.namedtuple(
 
 def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
     '''
-    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
-    temporary file in the given borgmatic runtime directory and return the file object so it can
-    continue to exist on disk as long as the caller needs it.
+    Given a sequence of patterns as Pattern instances, write them to a named temporary file in the
+    given borgmatic runtime directory and return the file object so it can continue to exist on disk
+    as long as the caller needs it.
 
     If an optional open pattern file is given, append to it instead of making a new temporary file.
     Return None if no patterns are provided.
@@ -70,7 +68,9 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=Non
         return None
 
     if patterns_file is None:
-        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
+        patterns_file = tempfile.NamedTemporaryFile(
+            'w', dir=borgmatic_runtime_directory, encoding='utf-8'
+        )
         operation_name = 'Writing'
     else:
         patterns_file.write('\n')
@@ -90,17 +90,17 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=Non
 
 def check_all_root_patterns_exist(patterns):
     '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
-    paths exist. If any don't, raise an exception.
+    Given a sequence of Pattern instances, check that all root pattern paths exist. If any don't,
+    raise an exception.
     '''
     missing_paths = [
         pattern.path
         for pattern in patterns
-        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+        if pattern.type == Pattern_type.ROOT
         if not os.path.exists(pattern.path)
     ]
 
     if missing_paths:
         raise ValueError(
-            f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}"
+            f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}",
         )

borgmatic/borg/prune.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -90,7 +91,7 @@ def prune_archives(
         + (('--list',) if config.get('list_details') else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 

borgmatic/borg/recreate.py

@@ -32,13 +32,15 @@ def recreate_archive(
     exclude_flags = flags.make_exclude_flags(config)
     compression = config.get('compression', None)
     chunker_params = config.get('chunker_params', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('recreate', '')
 
     # Available recompress MODES: "if-different", "always", "never" (default)
     recompress = config.get('recompress', None)
 
     # Write patterns to a temporary file and use that file with --patterns-from.
     patterns_file = write_patterns_file(
-        patterns, borgmatic.config.paths.get_working_directory(config)
+        patterns,
+        borgmatic.config.paths.get_working_directory(config),
     )
 
     recreate_command = (
@@ -70,6 +72,7 @@ def recreate_archive(
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--recompress', recompress) if recompress else ())
         + exclude_flags
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (
             (
                 flags.make_repository_flags(repository, local_borg_version)
@@ -80,7 +83,8 @@ def recreate_archive(
                 )
             )
             if borgmatic.borg.feature.available(
-                borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version
+                borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE,
+                local_borg_version,
             )
             else (
                 flags.make_repository_archive_flags(repository, archive, local_borg_version)

borgmatic/borg/rename.py

@@ -0,0 +1,68 @@
+import logging
+import shlex
+
+import borgmatic.borg.flags
+
+logger = logging.getLogger(__name__)
+
+
+def make_rename_command(
+    dry_run,
+    repository_name,
+    old_archive_name,
+    new_archive_name,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    extra_borg_options = config.get('extra_borg_options', {}).get('rename', '')
+
+    return (
+        (local_path, 'rename')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + borgmatic.borg.flags.make_flags('dry-run', dry_run)
+        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
+        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
+        + borgmatic.borg.flags.make_flags('log-json', config.get('log_json'))
+        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + borgmatic.borg.flags.make_repository_archive_flags(
+            repository_name,
+            old_archive_name,
+            local_borg_version,
+        )
+        + (new_archive_name,)
+    )
+
+
+def rename_archive(
+    repository_name,
+    old_archive_name,
+    new_archive_name,
+    dry_run,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    command = make_rename_command(
+        dry_run,
+        repository_name,
+        old_archive_name,
+        new_archive_name,
+        config,
+        local_borg_version,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.execute.execute_command(
+        command,
+        output_log_level=logging.INFO,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/repo_create.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import logging
+import shlex
 import subprocess
 
 import borgmatic.config.paths
@@ -48,13 +49,13 @@ def create_repository(
                 global_arguments,
                 local_path,
                 remote_path,
-            )
+            ),
         )
         repository_encryption_mode = info_data.get('encryption', {}).get('mode')
 
         if repository_encryption_mode != encryption_mode:
             raise ValueError(
-                f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"'
+                f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"',
             )
 
         logger.info('Repository already exists. Skipping creation.')
@@ -65,8 +66,14 @@ def create_repository(
 
     lock_wait = config.get('lock_wait')
     umask = config.get('umask')
+    extra_borg_options_from_init = config.get('extra_borg_options', {}).get('init', '')
     extra_borg_options = config.get('extra_borg_options', {}).get('repo-create', '')
 
+    if extra_borg_options_from_init:
+        logger.warning(
+            'The "init" option in "extra_borg_options" is deprecated and will be removed from a future release; use "repo_create" instead.'
+        )
+
     repo_create_command = (
         (local_path,)
         + (
@@ -86,12 +93,13 @@ def create_repository(
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options_from_init)) if extra_borg_options_from_init else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if dry_run:
-        logging.info('Skipping repository creation (dry run)')
+        logger.info('Skipping repository creation (dry run)')
         return
 
     # Do not capture output here, so as to support interactive prompts.

borgmatic/borg/repo_delete.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.borg.environment
 import borgmatic.borg.feature
@@ -9,6 +10,9 @@ import borgmatic.execute
 logger = logging.getLogger(__name__)
 
 
+FORCE_HARDER_FLAG_COUNT = 2
+
+
 def make_repo_delete_command(
     repository,
     config,
@@ -23,12 +27,22 @@ def make_repo_delete_command(
     arguments to the repo_delete action as an argparse.Namespace, and global arguments, return a command
     as a tuple to repo_delete the entire repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_delete'
+        if borgmatic.borg.feature.available(
+            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+        )
+        else 'delete',
+        '',
+    )
+
     return (
         (local_path,)
         + (
             ('repo-delete',)
             if borgmatic.borg.feature.available(
-                borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+                borgmatic.borg.feature.Feature.REPO_DELETE,
+                local_borg_version,
             )
             else ('delete',)
         )
@@ -41,13 +55,18 @@ def make_repo_delete_command(
         + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
         + borgmatic.borg.flags.make_flags('list', config.get('list_details'))
         + (
-            (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ()))
+            (
+                ('--force',)
+                + (('--force',) if repo_delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
+            )
             if repo_delete_arguments.force
             else ()
         )
         + borgmatic.borg.flags.make_flags_from_arguments(
-            repo_delete_arguments, excludes=('list_details', 'force', 'repository')
+            repo_delete_arguments,
+            excludes=('list_details', 'force', 'repository'),
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
     )
 

borgmatic/borg/repo_info.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -24,6 +25,10 @@ def display_repository_info(
     '''
     borgmatic.logger.add_custom_log_levels()
     lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_info' if feature.available(feature.Feature.REPO_INFO, local_borg_version) else 'info',
+        '',
+    )
 
     full_command = (
         (local_path,)
@@ -47,6 +52,7 @@ def display_repository_info(
         + flags.make_flags('log-json', config.get('log_json'))
         + flags.make_flags('lock-wait', lock_wait)
         + (('--json',) if repo_info_arguments.json else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
@@ -61,12 +67,14 @@ def display_repository_info(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
-    else:
-        execute_command(
-            full_command,
-            output_log_level=logging.ANSWER,
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
-        )
+
+    execute_command(
+        full_command,
+        output_log_level=logging.ANSWER,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )
+
+    return None

borgmatic/borg/repo_list.py

@@ -1,5 +1,7 @@
 import argparse
+import json
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -22,49 +24,93 @@ def resolve_archive_name(
     Given a local or remote repository path, an archive name, a configuration dict, the local Borg
     version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
     return the archive name. But if the archive name is "latest", then instead introspect the
-    repository for the latest archive and return its name.
+    repository for the latest archive and return its name or ID, depending on whether the version of
+    Borg in use supports archive series—different archives that share the same name but have unique
+    IDs.
 
     Raise ValueError if "latest" is given but there are no archives in the repository.
     '''
     if archive != 'latest':
         return archive
 
-    full_command = (
-        (
-            local_path,
-            (
-                'repo-list'
-                if feature.available(feature.Feature.REPO_LIST, local_borg_version)
-                else 'list'
-            ),
-        )
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', config.get('log_json'))
-        + flags.make_flags('lock-wait', config.get('lock_wait'))
-        + flags.make_flags('last', 1)
-        + ('--short',)
-        + flags.make_repository_flags(repository_path, local_borg_version)
+    latest_archive = get_latest_archive(
+        repository_path,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
     )
 
-    output = execute_command_and_capture_output(
+    return (
+        latest_archive['id']
+        if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
+        else latest_archive['name']
+    )
+
+
+def get_latest_archive(
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+    consider_checkpoints=False,
+):
+    '''
+    Returns a dict with information about the latest archive of a repository.
+
+    Raises ValueError if there are no archives in the repository.
+    '''
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_list' if feature.available(feature.Feature.REPO_LIST, local_borg_version) else 'list',
+        '',
+    )
+
+    full_command = (
+        local_path,
+        (
+            'repo-list'
+            if feature.available(feature.Feature.REPO_LIST, local_borg_version)
+            else 'list'
+        ),
+        *flags.make_flags('remote-path', remote_path),
+        *flags.make_flags('umask', config.get('umask')),
+        *flags.make_flags('log-json', config.get('log_json')),
+        *flags.make_flags('lock-wait', config.get('lock_wait')),
+        *(
+            flags.make_flags('consider-checkpoints', consider_checkpoints)
+            if not feature.available(feature.Feature.REPO_LIST, local_borg_version)
+            else ()
+        ),
+        *flags.make_flags('last', 1),
+        '--json',
+        *(tuple(shlex.split(extra_borg_options)) if extra_borg_options else ()),
+        *flags.make_repository_flags(repository_path, local_borg_version),
+    )
+
+    json_output = execute_command_and_capture_output(
         full_command,
         environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
+
+    archives = json.loads(json_output)['archives']
+
     try:
-        latest_archive = output.strip().splitlines()[-1]
+        latest_archive = archives[-1]
     except IndexError:
         raise ValueError('No archives found in the repository')
 
-    logger.debug(f'Latest archive is {latest_archive}')
+    logger.debug(f'Latest archive is {latest_archive["name"]} ({latest_archive["id"]})')
 
     return latest_archive
 
 
-MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
+MAKE_FLAGS_EXCLUDES = ('repository', 'format', 'prefix', 'match_archives')
 
 
 def make_repo_list_command(
@@ -81,6 +127,11 @@ def make_repo_list_command(
     arguments to the repo_list action, global arguments as an argparse.Namespace instance, and local and
     remote Borg paths, return a command as a tuple to list archives with a repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_list' if feature.available(feature.Feature.REPO_LIST, local_borg_version) else 'list',
+        '',
+    )
+
     return (
         (
             local_path,
@@ -119,7 +170,11 @@ def make_repo_list_command(
                 )
             )
         )
+        + flags.make_flags(
+            'format', repo_list_arguments.format or config.get('archive_list_format')
+        )
         + flags.make_flags_from_arguments(repo_list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
@@ -183,3 +238,5 @@ def list_repository(
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
+
+    return None

borgmatic/borg/state.py

@@ -1 +1 @@
-DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
+DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'  # pragma: nocover

borgmatic/borg/transfer.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -24,6 +25,7 @@ def transfer_archives(
     instance, transfer archives to the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
+    extra_borg_options = config.get('extra_borg_options', {}).get('transfer', '')
 
     full_command = (
         (local_path, 'transfer')
@@ -53,6 +55,7 @@ def transfer_archives(
                 )
             )
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
         + flags.make_flags('other-repo', transfer_arguments.source_repository)
         + flags.make_flags('dry-run', dry_run)

borgmatic/borg/umount.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.execute import execute_command
@@ -11,10 +12,12 @@ def unmount_archive(config, mount_point, local_path='borg'):
     Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
     from the mount point.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('umount', '')
     full_command = (
         (local_path, 'umount')
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (mount_point,)
     )
 

borgmatic/borg/version.py

@@ -19,6 +19,7 @@ def local_borg_version(config, local_path='borg'):
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
     )
+
     output = execute_command_and_capture_output(
         full_command,
         environment=environment.make_environment(config),

borgmatic/commands/arguments.py

@@ -60,7 +60,7 @@ def get_subactions_for_actions(action_parsers):
         action: tuple(
             subaction_name
             for group_action in action_parser._subparsers._group_actions
-            for subaction_name in group_action.choices.keys()
+            for subaction_name in group_action.choices
         )
         for action, action_parser in action_parsers.items()
         if action_parser._subparsers
@@ -77,21 +77,25 @@ def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments
     '''
     remaining_arguments = list(unparsed_arguments)
 
-    for action_name, parsed in parsed_arguments.items():
+    for parsed in parsed_arguments.values():
         for value in vars(parsed).values():
             if isinstance(value, str):
-                if value in ACTION_ALIASES.keys() and value in remaining_arguments:
+                if value in ACTION_ALIASES and value in remaining_arguments:
                     remaining_arguments.remove(value)
             elif isinstance(value, list):
                 for item in value:
-                    if item in ACTION_ALIASES.keys() and item in remaining_arguments:
+                    if item in ACTION_ALIASES and item in remaining_arguments:
                         remaining_arguments.remove(item)
 
     return tuple(remaining_arguments)
 
 
 def parse_and_record_action_arguments(
-    unparsed_arguments, parsed_arguments, action_parser, action_name, canonical_name=None
+    unparsed_arguments,
+    parsed_arguments,
+    action_parser,
+    action_name,
+    canonical_name=None,
 ):
     '''
     Given unparsed arguments as a sequence of strings, parsed arguments as a dict from action name
@@ -102,7 +106,8 @@ def parse_and_record_action_arguments(
     given action doesn't apply to the given unparsed arguments.
     '''
     filtered_arguments = omit_values_colliding_with_action_names(
-        unparsed_arguments, parsed_arguments
+        unparsed_arguments,
+        parsed_arguments,
     )
 
     if action_name not in filtered_arguments:
@@ -186,12 +191,12 @@ def get_unparsable_arguments(remaining_action_arguments):
         itertools.chain.from_iterable(
             argument_group
             for argument_group in dict.fromkeys(
-                itertools.chain.from_iterable(grouped_action_arguments)
-            ).keys()
+                itertools.chain.from_iterable(grouped_action_arguments),
+            )
             if all(
                 argument_group in action_arguments for action_arguments in grouped_action_arguments
             )
-        )
+        ),
     )
 
 
@@ -244,7 +249,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
                             subaction_name,
                         )
                         if argument != action_name
-                    )
+                    ),
                 )
 
                 if subaction_name in arguments:
@@ -256,14 +261,18 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
                     sys.exit(0)
                 else:
                     raise ValueError(
-                        f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}"
+                        f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}",
                     )
         # Otherwise, parse with the main action parser.
         else:
             remaining_action_arguments.append(
                 parse_and_record_action_arguments(
-                    unparsed_arguments, arguments, action_parser, action_name, canonical_name
-                )
+                    unparsed_arguments,
+                    arguments,
+                    action_parser,
+                    action_name,
+                    canonical_name,
+                ),
             )
 
     # If no actions were explicitly requested, assume defaults.
@@ -272,11 +281,11 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
             default_action_parser = action_parsers[default_action_name]
             remaining_action_arguments.append(
                 parse_and_record_action_arguments(
-                    tuple(unparsed_arguments) + (default_action_name,),
+                    (*unparsed_arguments, default_action_name),
                     arguments,
                     default_action_parser,
                     default_action_name,
-                )
+                ),
             )
 
     arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments)
@@ -288,7 +297,14 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
     )
 
 
-OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'}
+OMITTED_FLAG_NAMES = {
+    'match-archives',
+    'progress',
+    'statistics',
+    'list-details',
+    'file-list-format',
+    'archive-list-format',
+}
 
 
 def make_argument_description(schema, flag_name):
@@ -304,10 +320,10 @@ def make_argument_description(schema, flag_name):
 
     if '[0]' in flag_name:
         pieces.append(
-            ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).'
+            ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).',
         )
 
-    if example and schema_type in ('array', 'object'):
+    if example and schema_type in ('array', 'object'):  # noqa: PLR6201
         example_buffer = io.StringIO()
         yaml = ruamel.yaml.YAML(typ='safe')
         yaml.default_flow_style = True
@@ -387,7 +403,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name):
         if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name:
             continue
 
-        if action_registry_name in ('store_true', 'store_false'):
+        if action_registry_name in {'store_true', 'store_false'}:
             arguments_group.add_argument(
                 unparsed_flag_name,
                 action=action_registry_name,
@@ -408,7 +424,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name):
             )
 
 
-def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None):
+def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None):  # noqa: PLR0912
     '''
     Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of
     unparsed argument strings, convert the entire schema into corresponding command-line flags and
@@ -466,7 +482,10 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
         if properties:
             for name, child in properties.items():
                 add_arguments_from_schema(
-                    arguments_group, child, unparsed_arguments, names + (name,)
+                    arguments_group,
+                    child,
+                    unparsed_arguments,
+                    (*names, name),
                 )
 
             return
@@ -483,12 +502,15 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
                     arguments_group,
                     child,
                     unparsed_arguments,
-                    names[:-1] + (f'{names[-1]}[0]',) + (name,),
+                    (*names[:-1], f'{names[-1]}[0]', name),
                 )
         # If there aren't any children, then this is an array of scalars. Recurse accordingly.
         else:
             add_arguments_from_schema(
-                arguments_group, items, unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',)
+                arguments_group,
+                items,
+                unparsed_arguments,
+                (*names[:-1], f'{names[-1]}[0]'),
             )
 
     flag_name = '.'.join(names).replace('_', '-')
@@ -515,9 +537,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
         )
 
         if names[-1].startswith('no_'):
-            no_flag_name = '.'.join(names[:-1] + (names[-1][len('no_') :],)).replace('_', '-')
+            no_flag_name = '.'.join((*names[:-1], names[-1][len('no_') :])).replace('_', '-')
         else:
-            no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-')
+            no_flag_name = '.'.join((*names[:-1], 'no-' + names[-1])).replace('_', '-')
 
         arguments_group.add_argument(
             f'--{no_flag_name}',
@@ -545,7 +567,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
     add_array_element_arguments(arguments_group, unparsed_arguments, flag_name)
 
 
-def make_parsers(schema, unparsed_arguments):
+def make_parsers(schema, unparsed_arguments):  # noqa: PLR0915
     '''
     Given a configuration schema dict and unparsed arguments as a sequence of strings, build a
     global arguments parser, individual action parsers, and a combined parser containing both.
@@ -553,8 +575,7 @@ def make_parsers(schema, unparsed_arguments):
     ignoring actions, and the combined parser is handy for displaying help that includes everything:
     global flags, a list of actions, etc.
     '''
-    config_paths = collect.get_default_config_paths(expand_home=True)
-    unexpanded_config_paths = collect.get_default_config_paths(expand_home=False)
+    config_paths = collect.get_default_config_paths()
 
     # Using allow_abbrev=False here prevents the global parser from erroring about "ambiguous"
     # options like --encryption. Such options are intended for an action parser rather than the
@@ -567,7 +588,7 @@ def make_parsers(schema, unparsed_arguments):
         '--config',
         dest='config_paths',
         action='append',
-        help=f"Configuration filename or directory, can specify flag multiple times, defaults to: -c {' -c '.join(unexpanded_config_paths)}",
+        help='Configuration filename or directory, can specify flag multiple times, defaults to /etc/borgmatic/config.yaml, /etc/borgmatic.d, $XDG_CONFIG_HOME/borgmatic/config.yaml, and $XDG_CONFIG_HOME/borgmatic.d, where $XDG_CONFIG_HOME defaults to $HOME/.config',
     )
     global_group.add_argument(
         '-n',
@@ -626,8 +647,8 @@ def make_parsers(schema, unparsed_arguments):
     repo_create_parser = action_parsers.add_parser(
         'repo-create',
         aliases=ACTION_ALIASES['repo-create'],
-        help='Create a new, empty Borg repository',
-        description='Create a new, empty Borg repository',
+        help='Create a new, empty Borg repository (also known as "init")',
+        description='Create a new, empty Borg repository (also known as "init")',
         add_help=False,
     )
     repo_create_group = repo_create_parser.add_argument_group('repo-create arguments')
@@ -667,10 +688,13 @@ def make_parsers(schema, unparsed_arguments):
         dest='make_parent_directories',
         default=None,
         action='store_true',
-        help='Create any missing parent directories of the repository directory',
+        help='Create any missing parent directories of the repository directory [Borg 1.x only]',
     )
     repo_create_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     transfer_parser = action_parsers.add_parser(
@@ -712,7 +736,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Only transfer archives with names, hashes, or series matching this pattern',
     )
     transfer_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     transfer_group.add_argument(
         '--first',
@@ -720,7 +746,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Only transfer first N archives after other filters are applied',
     )
     transfer_group.add_argument(
-        '--last', metavar='N', help='Only transfer last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Only transfer last N archives after other filters are applied',
     )
     transfer_group.add_argument(
         '--oldest',
@@ -743,7 +771,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Transfer archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
     )
     transfer_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     prune_parser = action_parsers.add_parser(
@@ -833,7 +864,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Minimum saved space percentage threshold for compacting a segment, defaults to 10',
     )
     compact_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     create_parser = action_parsers.add_parser(
@@ -870,7 +904,16 @@ def make_parsers(schema, unparsed_arguments):
         help='Show per-file details',
     )
     create_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        dest='json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
+    )
+    create_group.add_argument(
+        '--comment',
+        metavar='COMMENT',
+        help='Add a comment text to the archive',
     )
     create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
@@ -991,13 +1034,19 @@ def make_parsers(schema, unparsed_arguments):
         help='Only delete archives with names, hashes, or series matching this pattern',
     )
     delete_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     delete_group.add_argument(
-        '--first', metavar='N', help='Delete first N archives after other filters are applied'
+        '--first',
+        metavar='N',
+        help='Delete first N archives after other filters are applied',
     )
     delete_group.add_argument(
-        '--last', metavar='N', help='Delete last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Delete last N archives after other filters are applied',
     )
     delete_group.add_argument(
         '--oldest',
@@ -1034,7 +1083,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to extract, defaults to the configured repository if there is only one, quoted globs supported',
     )
     extract_group.add_argument(
-        '--archive', help='Name or hash of a single archive to extract (or "latest")', required=True
+        '--archive',
+        help='Name or hash of a single archive to extract (or "latest")',
+        required=True,
     )
     extract_group.add_argument(
         '--path',
@@ -1063,7 +1114,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Display progress for each file as it is extracted',
     )
     extract_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     config_parser = action_parsers.add_parser(
@@ -1088,7 +1142,7 @@ def make_parsers(schema, unparsed_arguments):
         add_help=False,
     )
     config_bootstrap_group = config_bootstrap_parser.add_argument_group(
-        'config bootstrap arguments'
+        'config bootstrap arguments',
     )
     config_bootstrap_group.add_argument(
         '--repository',
@@ -1143,7 +1197,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Command to use instead of "ssh"',
     )
     config_bootstrap_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     config_generate_parser = config_parsers.add_parser(
@@ -1162,9 +1219,9 @@ def make_parsers(schema, unparsed_arguments):
     config_generate_group.add_argument(
         '-d',
         '--destination',
-        dest='destination_filename',
+        dest='destination_path',
         default=config_paths[0],
-        help=f'Destination configuration file, default: {unexpanded_config_paths[0]}',
+        help='Destination configuration file (or directory if using --split), default: /etc/borgmatic/config.yaml',
     )
     config_generate_group.add_argument(
         '--overwrite',
@@ -1173,7 +1230,15 @@ def make_parsers(schema, unparsed_arguments):
         help='Whether to overwrite any existing destination file, defaults to false',
     )
     config_generate_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '--split',
+        action='store_true',
+        help='Assuming the destination is a directory instead of a file, split the configuration into separate files within it, one per option, useful for documentation',
+    )
+    config_generate_group.add_argument(
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     config_validate_parser = config_parsers.add_parser(
@@ -1190,7 +1255,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Show the validated configuration after all include merging has occurred',
     )
     config_validate_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     export_tar_parser = action_parsers.add_parser(
@@ -1206,7 +1274,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to export from, defaults to the configured repository if there is only one, quoted globs supported',
     )
     export_tar_group.add_argument(
-        '--archive', help='Name or hash of a single archive to export (or "latest")', required=True
+        '--archive',
+        help='Name or hash of a single archive to export (or "latest")',
+        required=True,
     )
     export_tar_group.add_argument(
         '--path',
@@ -1223,7 +1293,8 @@ def make_parsers(schema, unparsed_arguments):
         required=True,
     )
     export_tar_group.add_argument(
-        '--tar-filter', help='Name of filter program to pipe data through'
+        '--tar-filter',
+        help='Name of filter program to pipe data through',
     )
     export_tar_group.add_argument(
         '--list',
@@ -1241,7 +1312,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Number of leading path components to remove from each exported path. Skip paths with fewer elements',
     )
     export_tar_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     mount_parser = action_parsers.add_parser(
@@ -1257,7 +1331,8 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to use, defaults to the configured repository if there is only one, quoted globs supported',
     )
     mount_group.add_argument(
-        '--archive', help='Name or hash of a single archive to mount (or "latest")'
+        '--archive',
+        help='Name or hash of a single archive to mount (or "latest")',
     )
     mount_group.add_argument(
         '--mount-point',
@@ -1286,7 +1361,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Mount first N archives after other filters are applied',
     )
     mount_group.add_argument(
-        '--last', metavar='N', help='Mount last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Mount last N archives after other filters are applied',
     )
     mount_group.add_argument(
         '--oldest',
@@ -1363,7 +1440,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Do not delete the local security info when deleting a repository',
     )
     repo_delete_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     restore_parser = action_parsers.add_parser(
@@ -1406,6 +1486,10 @@ def make_parsers(schema, unparsed_arguments):
         '--port',
         help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--container',
+        help='Container to restore to. Defaults to the "restore_container" option in borgmatic\'s configuration',
+    )
     restore_group.add_argument(
         '--username',
         help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
@@ -1418,10 +1502,18 @@ def make_parsers(schema, unparsed_arguments):
         '--restore-path',
         help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--original-label',
+        help='The label where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '--original-hostname',
         help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
     )
+    restore_group.add_argument(
+        '--original-container',
+        help='The container where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '--original-port',
         type=int,
@@ -1432,7 +1524,10 @@ def make_parsers(schema, unparsed_arguments):
         help='The name of the data source hook for the dump to restore, only necessary if you need to disambiguate dumps',
     )
     restore_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     repo_list_parser = action_parsers.add_parser(
@@ -1448,14 +1543,22 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to list, defaults to the configured repositories, quoted globs supported',
     )
     repo_list_group.add_argument(
-        '--short', default=False, action='store_true', help='Output only archive names'
+        '--short',
+        default=False,
+        action='store_true',
+        help='Output only archive names',
     )
-    repo_list_group.add_argument('--format', help='Format for archive listing')
+    repo_list_group.add_argument('--format', help='Borg format for the archive listing')
     repo_list_group.add_argument(
-        '--json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     repo_list_group.add_argument(
-        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
+        '-P',
+        '--prefix',
+        help='Deprecated. Only list archive names starting with this prefix',
     )
     repo_list_group.add_argument(
         '-a',
@@ -1465,13 +1568,19 @@ def make_parsers(schema, unparsed_arguments):
         help='Only list archive names, hashes, or series matching this pattern',
     )
     repo_list_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     repo_list_group.add_argument(
-        '--first', metavar='N', help='List first N archives after other filters are applied'
+        '--first',
+        metavar='N',
+        help='List first N archives after other filters are applied',
     )
     repo_list_group.add_argument(
-        '--last', metavar='N', help='List last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='List last N archives after other filters are applied',
     )
     repo_list_group.add_argument(
         '--oldest',
@@ -1500,7 +1609,10 @@ def make_parsers(schema, unparsed_arguments):
         help="List only deleted archives that haven't yet been compacted [Borg 2.x+ only]",
     )
     repo_list_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     list_parser = action_parsers.add_parser(
@@ -1516,7 +1628,8 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository containing archive to list, defaults to the configured repositories, quoted globs supported',
     )
     list_group.add_argument(
-        '--archive', help='Name or hash of a single archive to list (or "latest")'
+        '--archive',
+        help='Name or hash of a single archive to list (or "latest")',
     )
     list_group.add_argument(
         '--path',
@@ -1533,14 +1646,22 @@ def make_parsers(schema, unparsed_arguments):
         help='Partial path or pattern to search for and list across multiple archives, can specify flag multiple times',
     )
     list_group.add_argument(
-        '--short', default=False, action='store_true', help='Output only path names'
+        '--short',
+        default=False,
+        action='store_true',
+        help='Output only path names',
     )
-    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument('--format', help='Borg format for the file listing')
     list_group.add_argument(
-        '--json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     list_group.add_argument(
-        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
+        '-P',
+        '--prefix',
+        help='Deprecated. Only list archive names starting with this prefix',
     )
     list_group.add_argument(
         '-a',
@@ -1550,19 +1671,30 @@ def make_parsers(schema, unparsed_arguments):
         help='Only list archive names matching this pattern',
     )
     list_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     list_group.add_argument(
-        '--first', metavar='N', help='List first N archives after other filters are applied'
+        '--first',
+        metavar='N',
+        help='List first N archives after other filters are applied',
     )
     list_group.add_argument(
-        '--last', metavar='N', help='List last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='List last N archives after other filters are applied',
     )
     list_group.add_argument(
-        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+        '-e',
+        '--exclude',
+        metavar='PATTERN',
+        help='Exclude paths matching the pattern',
     )
     list_group.add_argument(
-        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+        '--exclude-from',
+        metavar='FILENAME',
+        help='Exclude paths from exclude file, one per line',
     )
     list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
     list_group.add_argument(
@@ -1585,10 +1717,17 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to show info for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     repo_info_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        dest='json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     repo_info_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     info_parser = action_parsers.add_parser(
@@ -1604,10 +1743,15 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     info_group.add_argument(
-        '--archive', help='Archive name, hash, or series to show info for (or "latest")'
+        '--archive',
+        help='Archive name, hash, or series to show info for (or "latest")',
     )
     info_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        dest='json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     info_group.add_argument(
         '-P',
@@ -1622,7 +1766,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Only show info for archive names, hashes, or series matching this pattern',
     )
     info_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     info_group.add_argument(
         '--first',
@@ -1630,7 +1776,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Show info for first N archives after other filters are applied',
     )
     info_group.add_argument(
-        '--last', metavar='N', help='Show info for last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Show info for last N archives after other filters are applied',
     )
     info_group.add_argument(
         '--oldest',
@@ -1667,7 +1815,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to break the lock for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     break_lock_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     key_parser = action_parsers.add_parser(
@@ -1712,7 +1863,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Path to export the key to, defaults to stdout (but be careful about dirtying the output with --verbosity)',
     )
     key_export_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     key_import_parser = key_parsers.add_parser(
@@ -1737,7 +1891,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Path to import the key from backup, defaults to stdin',
     )
     key_import_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     key_change_passphrase_parser = key_parsers.add_parser(
@@ -1747,14 +1904,17 @@ def make_parsers(schema, unparsed_arguments):
         add_help=False,
     )
     key_change_passphrase_group = key_change_passphrase_parser.add_argument_group(
-        'key change-passphrase arguments'
+        'key change-passphrase arguments',
     )
     key_change_passphrase_group.add_argument(
         '--repository',
         help='Path of repository to change the passphrase for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     key_change_passphrase_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     recreate_parser = action_parsers.add_parser(
@@ -1804,7 +1964,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Only consider archive names, hashes, or series matching this pattern [Borg 2.x+ only]',
     )
     recreate_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     borg_parser = action_parsers.add_parser(
@@ -1820,7 +1983,8 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported',
     )
     borg_group.add_argument(
-        '--archive', help='Archive name, hash, or series to pass to Borg (or "latest")'
+        '--archive',
+        help='Archive name, hash, or series to pass to Borg (or "latest")',
     )
     borg_group.add_argument(
         '--',
@@ -1834,6 +1998,9 @@ def make_parsers(schema, unparsed_arguments):
     return global_parser, action_parsers, global_plus_action_parser
 
 
+HIGHLANDER_ACTION_ARGUMENTS_COUNT = 2  # 1 for "global" + 1 for the action
+
+
 def parse_arguments(schema, *unparsed_arguments):
     '''
     Given a configuration schema dict and the command-line arguments with which this script was
@@ -1844,21 +2011,22 @@ def parse_arguments(schema, *unparsed_arguments):
     Raise SystemExit with an error code of 0 if "--help" was requested.
     '''
     global_parser, action_parsers, global_plus_action_parser = make_parsers(
-        schema, unparsed_arguments
+        schema,
+        unparsed_arguments,
     )
     arguments, remaining_action_arguments = parse_arguments_for_actions(
-        unparsed_arguments, action_parsers.choices, global_parser
+        unparsed_arguments,
+        action_parsers.choices,
+        global_parser,
     )
 
     if not arguments['global'].config_paths:
-        arguments['global'].config_paths = collect.get_default_config_paths(expand_home=True)
+        arguments['global'].config_paths = collect.get_default_config_paths()
 
     for action_name in ('bootstrap', 'generate', 'validate'):
-        if (
-            action_name in arguments.keys() and len(arguments.keys()) > 2
-        ):  # 2 = 1 for 'global' + 1 for the action
+        if action_name in arguments and len(arguments) > HIGHLANDER_ACTION_ARGUMENTS_COUNT:
             raise ValueError(
-                f'The {action_name} action cannot be combined with other actions. Please run it separately.'
+                f'The {action_name} action cannot be combined with other actions. Please run it separately.',
             )
 
     unknown_arguments = get_unparsable_arguments(remaining_action_arguments)
@@ -1870,11 +2038,11 @@ def parse_arguments(schema, *unparsed_arguments):
 
         global_plus_action_parser.print_usage()
         raise ValueError(
-            f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}"
+            f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}",
         )
 
     if (
-        ('list' in arguments and 'repo-info' in arguments and arguments['list'].json)
+        ('list' in arguments and 'repo-info' in arguments and arguments['list'].json)  # noqa: PLR0916
         or ('list' in arguments and 'info' in arguments and arguments['list'].json)
         or ('repo-info' in arguments and 'info' in arguments and arguments['repo-info'].json)
     ):
@@ -1882,23 +2050,23 @@ def parse_arguments(schema, *unparsed_arguments):
 
     if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives):
         raise ValueError(
-            'With the list action, only one of --prefix or --match-archives flags can be used.'
+            'With the list action, only one of --prefix or --match-archives flags can be used.',
         )
 
     if 'repo-list' in arguments and (
         arguments['repo-list'].prefix and arguments['repo-list'].match_archives
     ):
         raise ValueError(
-            'With the repo-list action, only one of --prefix or --match-archives flags can be used.'
+            'With the repo-list action, only one of --prefix or --match-archives flags can be used.',
         )
 
-    if 'info' in arguments and (
+    if 'info' in arguments and (  # noqa: PLR0916
         (arguments['info'].archive and arguments['info'].prefix)
         or (arguments['info'].archive and arguments['info'].match_archives)
         or (arguments['info'].prefix and arguments['info'].match_archives)
     ):
         raise ValueError(
-            'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.'
+            'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.',
         )
 
     if 'borg' in arguments and arguments['global'].dry_run:

borgmatic/commands/borgmatic.py

@@ -96,7 +96,7 @@ class Monitoring_hooks:
         self.config = config
         self.dry_run = global_arguments.dry_run
         self.monitoring_log_level = verbosity_to_log_level(
-            get_verbosity({config_filename: config}, 'monitoring_verbosity')
+            get_verbosity({config_filename: config}, 'monitoring_verbosity'),
         )
         self.monitoring_hooks_are_activated = (
             using_primary_action and self.monitoring_log_level != DISABLED
@@ -182,7 +182,7 @@ class Monitoring_hooks:
         )
 
 
-def run_configuration(config_filename, config, config_paths, arguments):
+def run_configuration(config_filename, config, config_paths, arguments):  # noqa: PLR0912, PLR0915
     '''
     Given a config filename, the corresponding parsed config dict, a sequence of loaded
     configuration paths, and command-line arguments as a dict from subparser name to a namespace of
@@ -206,12 +206,13 @@ def run_configuration(config_filename, config, config_paths, arguments):
 
     if skip_actions:
         logger.debug(
-            f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions"
+            f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions",
         )
 
-    try:
-        with Monitoring_hooks(config_filename, config, arguments, global_arguments):
-            with borgmatic.hooks.command.Before_after_hooks(
+    try:  # noqa: PLR1702
+        with (
+            Monitoring_hooks(config_filename, config, arguments, global_arguments),
+            borgmatic.hooks.command.Before_after_hooks(
                 command_hooks=config.get('commands'),
                 before_after='configuration',
                 umask=config.get('umask'),
@@ -220,80 +221,80 @@ def run_configuration(config_filename, config, config_paths, arguments):
                 action_names=arguments.keys(),
                 configuration_filename=config_filename,
                 log_file=config.get('log_file', ''),
-            ):
-                try:
-                    local_borg_version = borg_version.local_borg_version(config, local_path)
-                    logger.debug(f'Borg {local_borg_version}')
-                except (OSError, CalledProcessError, ValueError) as error:
-                    yield from log_error_records(
-                        f'{config_filename}: Error getting local Borg version', error
-                    )
-                    return
+            ),
+        ):
+            try:
+                local_borg_version = borg_version.local_borg_version(config, local_path)
+                logger.debug(f'Borg {local_borg_version}')
+            except (OSError, CalledProcessError, ValueError) as error:
+                yield from log_error_records(
+                    f'{config_filename}: Error getting local Borg version',
+                    error,
+                )
+                raise
 
-                for repo in config['repositories']:
-                    repo_queue.put(
-                        (repo, 0),
-                    )
+            for repo in config['repositories']:
+                repo_queue.put(
+                    (repo, 0),
+                )
 
-                while not repo_queue.empty():
-                    repository, retry_num = repo_queue.get()
+            while not repo_queue.empty():
+                repository, retry_num = repo_queue.get()
 
-                    with Log_prefix(repository.get('label', repository['path'])):
-                        logger.debug('Running actions for repository')
-                        timeout = retry_num * retry_wait
-                        if timeout:
-                            logger.warning(f'Sleeping {timeout}s before next retry')
-                            time.sleep(timeout)
-                        try:
-                            yield from run_actions(
-                                arguments=arguments,
-                                config_filename=config_filename,
-                                config=config,
-                                config_paths=config_paths,
-                                local_path=local_path,
-                                remote_path=remote_path,
-                                local_borg_version=local_borg_version,
-                                repository=repository,
+                with Log_prefix(repository.get('label', repository['path'])):
+                    logger.debug('Running actions for repository')
+                    timeout = retry_num * retry_wait
+
+                    if timeout:
+                        logger.warning(f'Sleeping {timeout}s before next retry')
+                        time.sleep(timeout)
+
+                    try:
+                        yield from run_actions(
+                            arguments=arguments,
+                            config_filename=config_filename,
+                            config=config,
+                            config_paths=config_paths,
+                            local_path=local_path,
+                            remote_path=remote_path,
+                            local_borg_version=local_borg_version,
+                            repository=repository,
+                        )
+                    except (OSError, CalledProcessError, ValueError) as error:
+                        if retry_num < retries:
+                            repo_queue.put(
+                                (repository, retry_num + 1),
                             )
-                        except (OSError, CalledProcessError, ValueError) as error:
-                            if retry_num < retries:
-                                repo_queue.put(
-                                    (repository, retry_num + 1),
-                                )
-                                tuple(  # Consume the generator so as to trigger logging.
-                                    log_error_records(
-                                        'Error running actions for repository',
-                                        error,
-                                        levelno=logging.WARNING,
-                                        log_command_error_output=True,
-                                    )
-                                )
-                                logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}')
-                                continue
-
-                            if command.considered_soft_failure(error):
-                                continue
-
-                            yield from log_error_records(
-                                'Error running actions for repository',
-                                error,
+                            tuple(  # Consume the generator so as to trigger logging.
+                                log_error_records(
+                                    'Error running actions for repository',
+                                    error,
+                                    levelno=logging.WARNING,
+                                    log_command_error_output=True,
+                                ),
                             )
-                            encountered_error = error
-                            error_repository = repository
+                            logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}')
+                            continue
 
-                # Re-raise any error, so that the Monitoring_hooks context manager wrapping this
-                # code can see the error and act accordingly. Do this here rather than as soon as
-                # the error is encountered so that an error with one repository doesn't prevent
-                # other repositories from running.
-                if encountered_error:
-                    raise encountered_error
+                        if command.considered_soft_failure(error):
+                            continue
+
+                        yield from log_error_records(
+                            'Error running actions for repository',
+                            error,
+                        )
+                        encountered_error = error
+                        error_repository = repository
+
+            # Re-raise any error, so that the Monitoring_hooks context manager wrapping this
+            # code can see the error and act accordingly. Do this here rather than as soon as
+            # the error is encountered so that an error with one repository doesn't prevent
+            # other repositories from running.
+            if encountered_error:
+                raise encountered_error
 
     except (OSError, CalledProcessError, ValueError) as error:
-        # No need to repeat logging of the error if it was already logged above.
-        if error_repository:
-            yield from log_error_records('Error running configuration')
-        else:
-            yield from log_error_records('Error running configuration', error)
+        yield from log_error_records('Error running configuration')
 
         encountered_error = error
 
@@ -325,7 +326,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
         yield from log_error_records(f'{config_filename}: Error running after error hook', error)
 
 
-def run_actions(
+def run_actions(  # noqa: PLR0912, PLR0915
     *,
     arguments,
     config_filename,
@@ -339,7 +340,7 @@ def run_actions(
     '''
     Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
     filename, a configuration dict, a sequence of loaded configuration paths, local and remote paths
-    to Borg, a local Borg version string, and a repository name, run all actions from the
+    to Borg, a local Borg version string, and a repository dict, run all actions from the
     command-line arguments on the given repository.
 
     Yield JSON output strings from executing any actions that produce JSON.
@@ -361,6 +362,22 @@ def run_actions(
         'repository': repository_path,
     }
     skip_actions = set(get_skip_actions(config, arguments))
+    requested_repository = next(
+        (
+            repository
+            for action_arguments in arguments.values()
+            for repository in (getattr(action_arguments, 'repository', None),)
+            if repository is not None
+        ),
+        None,
+    )
+
+    if requested_repository and not borgmatic.config.validate.repositories_match(
+        repository,
+        requested_repository,
+    ):
+        logger.debug('Skipping actions because the requested --repository does not match')
+        return
 
     with borgmatic.hooks.command.Before_after_hooks(
         command_hooks=config.get('commands'),
@@ -643,9 +660,9 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
                         levelno=logging.DEBUG,
                         levelname='DEBUG',
                         msg=f'{config_filename}: Loading configuration file',
-                    )
+                    ),
                 ),
-            ]
+            ],
         )
         try:
             configs[config_filename], paths, parse_logs = validate.parse_configuration(
@@ -662,12 +679,12 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
                 [
                     logging.makeLogRecord(
                         dict(
-                            levelno=logging.WARNING,
-                            levelname='WARNING',
+                            levelno=logging.CRITICAL,
+                            levelname='CRITICAL',
                             msg=f'{config_filename}: Insufficient permissions to read configuration file',
-                        )
+                        ),
                     ),
-                ]
+                ],
             )
         except (ValueError, OSError, validate.Validation_error) as error:
             logs.extend(
@@ -677,12 +694,12 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
                             levelno=logging.CRITICAL,
                             levelname='CRITICAL',
                             msg=f'{config_filename}: Error parsing configuration file',
-                        )
+                        ),
                     ),
                     logging.makeLogRecord(
-                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=str(error))
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=str(error)),
                     ),
-                ]
+                ],
             )
 
     return (configs, sorted(config_paths), logs)
@@ -705,7 +722,10 @@ BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE = 62
 
 
 def log_error_records(
-    message, error=None, levelno=logging.CRITICAL, log_command_error_output=False
+    message,
+    error=None,
+    levelno=logging.CRITICAL,
+    log_command_error_output=False,
 ):
     '''
     Given error message text, an optional exception object, an optional log level, and whether to
@@ -723,14 +743,14 @@ def log_error_records(
 
     try:
         raise error
-    except CalledProcessError as error:
+    except CalledProcessError as called_process_error:
         yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
 
-        if error.output:
+        if called_process_error.output:
             try:
-                output = error.output.decode('utf-8')
+                output = called_process_error.output.decode('utf-8')
             except (UnicodeDecodeError, AttributeError):
-                output = error.output
+                output = called_process_error.output
 
             # Suppress these logs for now and save the error output for the log summary at the end.
             # Log a separate record per line, as some errors can be really verbose and overflow the
@@ -743,18 +763,18 @@ def log_error_records(
                     suppress_log=True,
                 )
 
-        yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(called_process_error))
 
-        if error.returncode == BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE:
+        if called_process_error.returncode == BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE:
             yield log_record(
                 levelno=levelno,
                 levelname=level_name,
                 msg='\nTo work around this, set either the "relocated_repo_access_is_ok" or "unknown_unencrypted_repo_access_is_ok" option to "true", as appropriate.',
             )
-    except (ValueError, OSError) as error:
+    except (ValueError, OSError) as other_error:
         yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
-        yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
-    except:  # noqa: E722
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(other_error))
+    except:  # noqa: E722, S110
         # Raising above only as a means of determining the error type. Swallow the exception here
         # because we don't want the exception to propagate out of this function.
         pass
@@ -785,7 +805,8 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
         try:
             # No configuration file is needed for bootstrap.
             local_borg_version = borg_version.local_borg_version(
-                {}, arguments['bootstrap'].local_path
+                {},
+                arguments['bootstrap'].local_path,
             )
         except (OSError, CalledProcessError, ValueError) as error:
             yield from log_error_records('Error getting local Borg version', error)
@@ -793,14 +814,16 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
 
         try:
             borgmatic.actions.config.bootstrap.run_bootstrap(
-                arguments['bootstrap'], arguments['global'], local_borg_version
+                arguments['bootstrap'],
+                arguments['global'],
+                local_borg_version,
             )
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.ANSWER,
                     levelname='ANSWER',
                     msg='Bootstrap successful',
-                )
+                ),
             )
         except (
             CalledProcessError,
@@ -814,14 +837,15 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
     if 'generate' in arguments:
         try:
             borgmatic.actions.config.generate.run_generate(
-                arguments['generate'], arguments['global']
+                arguments['generate'],
+                arguments['global'],
             )
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.ANSWER,
                     levelname='ANSWER',
                     msg='Generate successful',
-                )
+                ),
             )
         except (
             CalledProcessError,
@@ -839,7 +863,7 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
                     levelno=logging.CRITICAL,
                     levelname='CRITICAL',
                     msg='Configuration validation failed',
-                )
+                ),
             )
 
             return
@@ -852,7 +876,7 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
                     levelno=logging.ANSWER,
                     levelname='ANSWER',
                     msg='All configuration files are valid',
-                )
+                ),
             )
         except (
             CalledProcessError,
@@ -864,7 +888,7 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
         return
 
 
-def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path):
+def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path):  # noqa: PLR0912
     '''
     Given a dict of configuration filename to corresponding parsed configuration, a sequence of
     loaded configuration paths, parsed command-line arguments as a dict from subparser name to a
@@ -877,9 +901,9 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
     # Run cross-file validation checks.
     repository = None
 
-    for action_name, action_arguments in arguments.items():
+    for action_arguments in arguments.values():
         if hasattr(action_arguments, 'repository'):
-            repository = getattr(action_arguments, 'repository')
+            repository = action_arguments.repository
             break
 
     try:
@@ -944,7 +968,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
                         levelno=logging.INFO,
                         levelname='INFO',
                         msg=f'{config_filename}: Successfully ran configuration file',
-                    )
+                    ),
                 )
                 if results:
                     json_results.extend(results)
@@ -1033,17 +1057,17 @@ def get_singular_option_value(configs, option_name):
         configure_logging(logging.CRITICAL)
         joined_values = ', '.join(str(value) for value in distinct_values)
         logger.critical(
-            f'The {option_name} option has conflicting values across configuration files: {joined_values}'
+            f'The {option_name} option has conflicting values across configuration files: {joined_values}',
         )
         exit_with_help_link()
 
     try:
-        return tuple(distinct_values)[0]
-    except IndexError:
+        return next(iter(distinct_values))
+    except StopIteration:
         return None
 
 
-def main(extra_summary_logs=[]):  # pragma: no cover
+def main(extra_summary_logs=()):  # pragma: no cover
     configure_signals()
     configure_delayed_logging()
     schema_filename = validate.schema_filename()
@@ -1064,19 +1088,23 @@ def main(extra_summary_logs=[]):  # pragma: no cover
     except SystemExit as error:
         if error.code == 0:
             raise error
+
         configure_logging(logging.CRITICAL)
         logger.critical(f"Error parsing arguments: {' '.join(sys.argv)}")
         exit_with_help_link()
 
     global_arguments = arguments['global']
+
     if global_arguments.version:
-        print(importlib.metadata.version('borgmatic'))
+        print(importlib.metadata.version('borgmatic'))  # noqa: T201
         sys.exit(0)
+
     if global_arguments.bash_completion:
-        print(borgmatic.commands.completion.bash.bash_completion())
+        print(borgmatic.commands.completion.bash.bash_completion())  # noqa: T201
         sys.exit(0)
+
     if global_arguments.fish_completion:
-        print(borgmatic.commands.completion.fish.fish_completion())
+        print(borgmatic.commands.completion.fish.fish_completion())  # noqa: T201
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
@@ -1115,18 +1143,23 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         exit_with_help_link()
 
     summary_logs = (
-        extra_summary_logs
+        list(extra_summary_logs)
         + parse_logs
         + (
             list(
                 collect_highlander_action_summary_logs(
-                    configs, arguments, configuration_parse_errors
-                )
+                    configs,
+                    arguments,
+                    configuration_parse_errors,
+                ),
             )
             or list(
                 collect_configuration_run_summary_logs(
-                    configs, config_paths, arguments, log_file_path
-                )
+                    configs,
+                    config_paths,
+                    arguments,
+                    log_file_path,
+                ),
             )
         )
     )

borgmatic/commands/completion/actions.py

@@ -22,15 +22,15 @@ def available_actions(subparsers, current_action=None):
     action of "config" but not "list".
     '''
     action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions(
-        subparsers.choices
+        subparsers.choices,
     )
     current_subactions = action_to_subactions.get(current_action)
 
     if current_subactions:
         return current_subactions
 
-    all_subactions = set(
+    all_subactions = {
         subaction for subactions in action_to_subactions.values() for subaction in subactions
-    )
+    }
 
-    return tuple(action for action in subparsers.choices.keys() if action not in all_subactions)
+    return tuple(action for action in subparsers.choices if action not in all_subactions)

borgmatic/commands/completion/bash.py

@@ -23,7 +23,7 @@ def bash_completion():
     borgmatic's command-line argument parsers.
     '''
     (
-        unused_global_parser,
+        _,
         action_parsers,
         global_plus_action_parser,
     ) = borgmatic.commands.arguments.make_parsers(
@@ -33,6 +33,7 @@ def bash_completion():
     global_flags = parser_flags(global_plus_action_parser)
 
     # Avert your eyes.
+    # fmt: off
     return '\n'.join(
         (
             'check_version() {',
@@ -47,24 +48,22 @@ def bash_completion():
             '    fi',
             '}',
             'complete_borgmatic() {',
-        )
-        + tuple(
+            *tuple(
             '''    if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
         COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
         return 0
-    fi'''
+    fi'''  # noqa: UP031
             % (
                 action,
                 parser_flags(action_parser),
                 ' '.join(
-                    borgmatic.commands.completion.actions.available_actions(action_parsers, action)
+                    borgmatic.commands.completion.actions.available_actions(action_parsers, action),
                 ),
                 global_flags,
             )
             for action, action_parser in reversed(action_parsers.choices.items())
-        )
-        + (
-            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
+        ),
+            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: UP031
             % (
                 ' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)),
                 global_flags,
@@ -72,5 +71,5 @@ def bash_completion():
             '    (check_version &)',
             '}',
             '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
-        )
+        ),
     )

borgmatic/commands/completion/fish.py

@@ -11,10 +11,10 @@ def has_file_options(action: Action):
     '''
     Given an argparse.Action instance, return True if it takes a file argument.
     '''
-    return action.metavar in (
+    return action.metavar in {
         'FILENAME',
         'PATH',
-    ) or action.dest in ('config_paths',)
+    } or action.dest in {'config_paths'}
 
 
 def has_choice_options(action: Action):
@@ -36,11 +36,11 @@ def has_unknown_required_param_options(action: Action):
     return (
         action.required is True
         or action.nargs
-        in (
+        in {
             '+',
             '*',
-        )
-        or action.metavar in ('PATTERN', 'KEYS', 'N')
+        }
+        or action.metavar in {'PATTERN', 'KEYS', 'N'}
         or (action.type is not None and action.default is None)
     )
 
@@ -77,7 +77,7 @@ def exact_options_completion(action: Action):
         return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"'''
 
     raise ValueError(
-        f'Unexpected action: {action} passes has_exact_options but has no choices produced'
+        f'Unexpected action: {action} passes has_exact_options but has no choices produced',
     )
 
 
@@ -96,7 +96,7 @@ def fish_completion():
     borgmatic's command-line argument parsers.
     '''
     (
-        unused_global_parser,
+        _,
         action_parsers,
         global_plus_action_parser,
     ) = borgmatic.commands.arguments.make_parsers(
@@ -104,7 +104,7 @@ def fish_completion():
         unparsed_arguments=(),
     )
 
-    all_action_parsers = ' '.join(action for action in action_parsers.choices.keys())
+    all_action_parsers = ' '.join(action for action in action_parsers.choices)
 
     exact_option_args = tuple(
         ' '.join(action.option_strings)
@@ -119,8 +119,9 @@ def fish_completion():
     )
 
     # Avert your eyes.
-    return '\n'.join(
-        dedent_strip_as_tuple(
+    # fmt: off
+    return '\n'.join((
+        *dedent_strip_as_tuple(
             f'''
             function __borgmatic_check_version
                 set -fx this_filename (status current-filename)
@@ -157,27 +158,27 @@ def fish_completion():
 
             set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}"
             set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
-            '''
-        )
-        + ('\n# action_parser completions',)
-        + tuple(
+            ''',
+        ),
+        '\n# action_parser completions',
+        *tuple(
             f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}'''
             for action_name, action_parser in action_parsers.choices.items()
-        )
-        + ('\n# global flags',)
-        + tuple(
+        ),
+        '\n# global flags',
+        *tuple(
             # -n is checked in order, so put faster / more likely to be true checks first
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
             for action in global_plus_action_parser._actions
             # ignore the noargs action, as this is an impossible completion for fish
             if len(action.option_strings) > 0
             if 'Deprecated' not in action.help
-        )
-        + ('\n# action_parser flags',)
-        + tuple(
+        ),
+        '\n# action_parser flags',
+        *tuple(
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
             for action_name, action_parser in action_parsers.choices.items()
             for action in action_parser._actions
             if 'Deprecated' not in (action.help or ())
-        )
-    )
+        ),
+    ))

borgmatic/commands/completion/flag.py

@@ -5,7 +5,7 @@ def variants(flag_name):
     "--foo[9].bar".
     '''
     if '[0]' in flag_name:
-        for index in range(0, 10):
+        for index in range(10):
             yield flag_name.replace('[0]', f'[{index}]')
 
         return

borgmatic/commands/generate_config.py

@@ -10,8 +10,8 @@ def main():
             levelno=logging.WARNING,
             levelname='WARNING',
             msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
-        )
+        ),
     )
 
-    sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
+    sys.argv = ['borgmatic', 'config', 'generate', *sys.argv[1:]]
     borgmatic.commands.borgmatic.main([warning_log])

borgmatic/commands/validate_config.py

@@ -10,8 +10,8 @@ def main():
             levelno=logging.WARNING,
             levelname='WARNING',
             msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.',
-        )
+        ),
     )
 
-    sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:]
+    sys.argv = ['borgmatic', 'config', 'validate', *sys.argv[1:]]
     borgmatic.commands.borgmatic.main([warning_log])

borgmatic/config/arguments.py

@@ -146,16 +146,16 @@ def prepare_arguments_for_config(global_arguments, schema):
         keys = tuple(argument_name.split('.'))
         option_type = type_for_option(schema, keys)
 
-        # The argument doesn't correspond to any option in the schema, so ignore it. It's
-        # probably a flag that borgmatic has on the command-line but not in configuration.
-        if option_type is None:
+        # The argument doesn't correspond to any option in the schema, or it is a complex argument, so ignore it.
+        # It's probably a flag that borgmatic has on the command-line but not in configuration.
+        if option_type in {'object', None}:
             continue
 
         prepared_values.append(
             (
                 keys,
                 convert_value_type(value, option_type),
-            )
+            ),
         )
 
     return tuple(prepared_values)

borgmatic/config/collect.py

@@ -1,17 +1,14 @@
 import os
 
 
-def get_default_config_paths(expand_home=True):
+def get_default_config_paths():
     '''
-    Based on the value of the XDG_CONFIG_HOME and HOME environment variables, return a list of
-    default configuration paths. This includes both system-wide configuration and configuration in
-    the current user's home directory.
-
-    Don't expand the home directory ($HOME) if the expand home flag is False.
+    Return a list of default configuration paths. This includes both system-wide
+    configuration and configuration in the current user's home directory.
     '''
-    user_config_directory = os.getenv('XDG_CONFIG_HOME') or os.path.join('$HOME', '.config')
-    if expand_home:
-        user_config_directory = os.path.expandvars(user_config_directory)
+    user_config_directory = os.path.expandvars(
+        os.getenv('XDG_CONFIG_HOME') or os.path.join('$HOME', '.config')
+    )
 
     return [
         '/etc/borgmatic/config.yaml',
@@ -49,6 +46,6 @@ def collect_config_filenames(config_paths):
 
         for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
-            matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
+            matching_filetype = full_filename.endswith(('.yaml', '.yml'))
             if matching_filetype and not os.path.isdir(full_filename):
                 yield os.path.abspath(full_filename)

borgmatic/config/constants.py

@@ -1,3 +1,4 @@
+import contextlib
 import shlex
 
 
@@ -6,17 +7,18 @@ def coerce_scalar(value):
     Given a configuration value, coerce it to an integer or a boolean as appropriate and return the
     result.
     '''
-    try:
+    with contextlib.suppress(TypeError, ValueError):
         return int(value)
-    except (TypeError, ValueError):
-        pass
 
-    if value == 'true' or value == 'True':
-        return True
-    if value == 'false' or value == 'False':
-        return False
-
-    return value
+    try:
+        return {
+            'true': True,
+            'True': True,
+            'false': False,
+            'False': False,
+        }.get(value, value)
+    except TypeError:  # e.g. for an unhashable type
+        return value
 
 
 def apply_constants(value, constants, shell_escape=False):
@@ -55,8 +57,7 @@ def apply_constants(value, constants, shell_escape=False):
                 constants,
                 shell_escape=(
                     shell_escape
-                    or option_name.startswith('before_')
-                    or option_name.startswith('after_')
+                    or option_name.startswith(('before_', 'after_'))
                     or option_name == 'on_error'
                 ),
             )

borgmatic/config/environment.py

@@ -2,7 +2,7 @@ import os
 import re
 
 VARIABLE_PATTERN = re.compile(
-    r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
+    r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})',
 )
 
 

borgmatic/config/generate.py

@@ -1,5 +1,7 @@
 import collections
+import contextlib
 import io
+import json
 import os
 import re
 
@@ -18,7 +20,8 @@ def insert_newline_before_comment(config, field_name):
     field and its comments.
     '''
     config.ca.items[field_name][1].insert(
-        0, ruamel.yaml.tokens.CommentToken('\n', ruamel.yaml.error.CommentMark(0), None)
+        0,
+        ruamel.yaml.tokens.CommentToken('\n', ruamel.yaml.error.CommentMark(0), None),
     )
 
 
@@ -40,13 +43,17 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
         config = ruamel.yaml.comments.CommentedSeq(
             example
             if borgmatic.config.schema.compare_types(
-                schema['items'].get('type'), SCALAR_SCHEMA_TYPES
+                schema['items'].get('type'),
+                SCALAR_SCHEMA_TYPES,
             )
             else [
                 schema_to_sample_configuration(
-                    schema['items'], source_config, level, parent_is_sequence=True
-                )
-            ]
+                    schema['items'],
+                    source_config,
+                    level,
+                    parent_is_sequence=True,
+                ),
+            ],
         )
         add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
     elif borgmatic.config.schema.compare_types(schema_type, {'object'}):
@@ -59,19 +66,25 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
                     (
                         field_name,
                         schema_to_sample_configuration(
-                            sub_schema, (source_config or {}).get(field_name, {}), level + 1
+                            sub_schema,
+                            (source_config or {}).get(field_name, {}),
+                            level + 1,
                         ),
                     )
                     for field_name, sub_schema in borgmatic.config.schema.get_properties(
-                        schema
+                        schema,
                     ).items()
-                ]
+                ],
             )
             or example
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
         add_comments_to_configuration_object(
-            config, schema, source_config, indent=indent, skip_first_field=parent_is_sequence
+            config,
+            schema,
+            source_config,
+            indent=indent,
+            skip_first_field=parent_is_sequence,
         )
     elif borgmatic.config.schema.compare_types(schema_type, SCALAR_SCHEMA_TYPES, match=all):
         return example
@@ -95,7 +108,7 @@ def comment_out_line(line):
     return '# '.join((indent_spaces, line[count_indent_spaces:]))
 
 
-def comment_out_optional_configuration(rendered_config):
+def transform_optional_configuration(rendered_config, comment_out=True):
     '''
     Post-process a rendered configuration string to comment out optional key/values, as determined
     by a sentinel in the comment before each key.
@@ -105,6 +118,9 @@ def comment_out_optional_configuration(rendered_config):
 
     Ideally ruamel.yaml would support commenting out keys during configuration generation, but it's
     not terribly easy to accomplish that way.
+
+    If comment_out is False, then just strip the comment sentinel without actually commenting
+    anything out.
     '''
     lines = []
     optional = False
@@ -117,16 +133,15 @@ def comment_out_optional_configuration(rendered_config):
         # Upon encountering an optional configuration option, comment out lines until the next blank
         # line.
         if line.strip().startswith(f'# {COMMENTED_OUT_SENTINEL}'):
+            if comment_out is False:
+                continue
+
             optional = True
             indent_characters_at_sentinel = indent_characters
             continue
 
-        # Hit a blank line, so reset commenting.
-        if not line.strip():
-            optional = False
-            indent_characters_at_sentinel = None
-        # Dedented, so reset commenting.
-        elif (
+        # Hit a blank line or dedented, so reset commenting.
+        if not line.strip() or (
             indent_characters_at_sentinel is not None
             and indent_characters < indent_characters_at_sentinel
         ):
@@ -158,15 +173,13 @@ def write_configuration(config_filename, rendered_config, mode=0o600, overwrite=
     '''
     if not overwrite and os.path.exists(config_filename):
         raise FileExistsError(
-            f'{config_filename} already exists. Aborting. Use --overwrite to replace the file.'
+            f'{config_filename} already exists. Aborting. Use --overwrite to replace the file.',
         )
 
-    try:
+    with contextlib.suppress(FileExistsError, FileNotFoundError):
         os.makedirs(os.path.dirname(config_filename), mode=0o700)
-    except (FileExistsError, FileNotFoundError):
-        pass
 
-    with open(config_filename, 'w') as config_file:
+    with open(config_filename, 'w', encoding='utf-8') as config_file:
         config_file.write(rendered_config)
 
     os.chmod(config_filename, mode)
@@ -191,7 +204,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
     if schema['items'].get('type') != 'object':
         return
 
-    for field_name in config[0].keys():
+    for field_name in config[0]:
         field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {})
         description = field_schema.get('description')
 
@@ -211,7 +224,11 @@ COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
 
 
 def add_comments_to_configuration_object(
-    config, schema, source_config=None, indent=0, skip_first_field=False
+    config,
+    schema,
+    source_config=None,
+    indent=0,
+    skip_first_field=False,
 ):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
@@ -239,7 +256,7 @@ def add_comments_to_configuration_object(
             source_config is None or field_name not in source_config
         ):
             description = (
-                '\n'.join((description, COMMENTED_OUT_SENTINEL))
+                f'{description}\n{COMMENTED_OUT_SENTINEL}'
                 if description
                 else COMMENTED_OUT_SENTINEL
             )
@@ -267,6 +284,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
     '''
     if not source_config:
         return destination_config
+
     if not destination_config or not isinstance(source_config, collections.abc.Mapping):
         return source_config
 
@@ -274,7 +292,8 @@ def merge_source_configuration_into_destination(destination_config, source_confi
         # This is a mapping. Recurse for this key/value.
         if isinstance(source_value, collections.abc.Mapping):
             destination_config[field_name] = merge_source_configuration_into_destination(
-                destination_config[field_name], source_value
+                destination_config[field_name],
+                source_value,
             )
             continue
 
@@ -288,27 +307,51 @@ def merge_source_configuration_into_destination(destination_config, source_confi
                         source_item,
                     )
                     for index, source_item in enumerate(source_value)
-                ]
+                ],
             )
             continue
 
         # This is some sort of scalar. Set it into the destination.
-        destination_config[field_name] = source_config[field_name]
+        destination_config[field_name] = source_value
 
     return destination_config
 
 
+def get_configuration_subset(config, option_name):  # pragma: no cover
+    '''
+    Given configuration as a ruamel.yaml.CommentedMap and an option name found within it at the top
+    level, return a new CommentedMap containing a subset of the configuration with only the given
+    option and no other top-level options.
+
+    This is useful when generating the sample configuration for a single option instead of a whole
+    configuration file.
+    '''
+    option_config = ruamel.yaml.CommentedMap({option_name: config[option_name]})
+
+    # Due to a quirk of ruamel.yaml, the comment right before a top-level key is not on that key and
+    # needs to get copied separately.
+    option_config.ca.items[option_name] = config.ca.items[option_name]
+
+    return option_config
+
+
 def generate_sample_configuration(
-    dry_run, source_filename, destination_filename, schema_filename, overwrite=False
+    dry_run,
+    source_filename,
+    destination_path,
+    schema_filename,
+    overwrite=False,
+    split=False,
 ):
     '''
-    Given an optional source configuration filename, and a required destination configuration
-    filename, the path to a schema filename in a YAML rendition of the JSON Schema format, and
-    whether to overwrite a destination file, write out a sample configuration file based on that
-    schema. If a source filename is provided, merge the parsed contents of that configuration into
-    the generated configuration.
+    Given an optional source configuration filename, a required destination configuration path, the
+    path to a schema filename in a YAML rendition of the JSON Schema format, whether to overwrite a
+    destination file, and whether to split the configuration into multiple files (one per option) in
+    the assumed destination directory, write out sample configuration file(s) based on that schema.
+    If a source filename is provided, merge the parsed contents of that configuration into the
+    generated configuration.
     '''
-    schema = ruamel.yaml.YAML(typ='safe').load(open(schema_filename))
+    schema = ruamel.yaml.YAML(typ='safe').load(open(schema_filename, encoding='utf-8'))
     source_config = None
 
     if source_filename:
@@ -322,14 +365,44 @@ def generate_sample_configuration(
             del source_config['bootstrap']
 
     destination_config = merge_source_configuration_into_destination(
-        schema_to_sample_configuration(schema, source_config), source_config
+        schema_to_sample_configuration(schema, source_config),
+        source_config,
     )
 
     if dry_run:
         return
 
+    if split:
+        if os.path.exists(destination_path) and not os.path.isdir(destination_path):
+            raise ValueError('With the --split flag, the destination path must be a directory')
+
+        os.makedirs(destination_path, exist_ok=True)
+
+        for option_name in destination_config:
+            write_configuration(
+                os.path.join(destination_path, f'{option_name}.yaml'),
+                transform_optional_configuration(
+                    render_configuration(get_configuration_subset(destination_config, option_name)),
+                    comment_out=False,
+                ).strip(),
+                overwrite=overwrite,
+            )
+
+        # Also dump a manifest listing all the options we've written.
+        json.dump(
+            {'option_names': list(destination_config.keys())},
+            open(os.path.join(destination_path, 'options.json'), 'w', encoding='utf-8'),
+        )
+
+        return
+
+    if os.path.exists(destination_path) and not os.path.isfile(destination_path):
+        raise ValueError('Without the --split flag, the destination path must be a file')
+
     write_configuration(
-        destination_filename,
-        comment_out_optional_configuration(render_configuration(destination_config)),
+        destination_path,
+        transform_optional_configuration(
+            render_configuration(destination_config), comment_out=True
+        ),
         overwrite=overwrite,
     )

borgmatic/config/load.py

@@ -31,7 +31,7 @@ def probe_and_include_file(filename, include_directories, config_paths):
             return load_configuration(candidate_filename, config_paths)
 
     raise FileNotFoundError(
-        f'Could not find include {filename} at {" or ".join(candidate_filenames)}'
+        f'Could not find include {filename} at {" or ".join(candidate_filenames)}',
     )
 
 
@@ -69,7 +69,7 @@ def include_configuration(loader, filename_node, include_directory, config_paths
         ]
 
     raise ValueError(
-        'The value given for the !include tag is invalid; use a single filename or a list of filenames instead'
+        'The value given for the !include tag is invalid; use a single filename or a list of filenames instead',
     )
 
 
@@ -85,7 +85,7 @@ def raise_retain_node_error(loader, node):
     '''
     if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)):
         raise ValueError(
-            'The !retain tag may only be used within a configuration file containing a merged !include tag.'
+            'The !retain tag may only be used within a configuration file containing a merged !include tag.',
         )
 
     raise ValueError('The !retain tag may only be used on a mapping or list.')
@@ -100,7 +100,7 @@ def raise_omit_node_error(loader, node):
     tags are handled by deep_merge_nodes() below.
     '''
     raise ValueError(
-        'The !omit tag may only be used on a scalar (e.g., string) or list element within a configuration file containing a merged !include tag.'
+        'The !omit tag may only be used on a scalar (e.g., string) or list element within a configuration file containing a merged !include tag.',
     )
 
 
@@ -111,9 +111,13 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
 
     def __init__(
-        self, preserve_quotes=None, loader=None, include_directory=None, config_paths=None
+        self,
+        preserve_quotes=None,
+        loader=None,
+        include_directory=None,
+        config_paths=None,
     ):
-        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        super().__init__(preserve_quotes, loader)
         self.add_constructor(
             '!include',
             functools.partial(
@@ -147,7 +151,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
         representer = ruamel.yaml.representer.SafeRepresenter()
 
         for index, (key_node, value_node) in enumerate(node.value):
-            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+            if key_node.tag == 'tag:yaml.org,2002:merge' and value_node.tag == '!include':
                 # Replace the merge include with a sequence of included configuration nodes ready
                 # for merging. The construct_object() call here triggers include_configuration()
                 # among other constructors.
@@ -157,7 +161,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
                 )
 
         # This super().flatten_mapping() call actually performs "<<" merges.
-        super(Include_constructor, self).flatten_mapping(node)
+        super().flatten_mapping(node)
 
         node.value = deep_merge_nodes(node.value)
 
@@ -179,7 +183,7 @@ def load_configuration(filename, config_paths=None):
     # because yaml.Constructor has to be an actual class.)
     class Include_constructor_with_extras(Include_constructor):
         def __init__(self, preserve_quotes=None, loader=None):
-            super(Include_constructor_with_extras, self).__init__(
+            super().__init__(
                 preserve_quotes,
                 loader,
                 include_directory=os.path.dirname(filename),
@@ -190,7 +194,7 @@ def load_configuration(filename, config_paths=None):
     yaml.Constructor = Include_constructor_with_extras
     config_paths.add(filename)
 
-    with open(filename) as file:
+    with open(filename, encoding='utf-8') as file:
         return yaml.load(file.read())
 
 
@@ -318,17 +322,18 @@ def deep_merge_nodes(nodes):
 
     # Bucket the nodes by their keys. Then merge all of the values sharing the same key.
     for key_name, grouped_nodes in itertools.groupby(
-        sorted(nodes, key=get_node_key_name), get_node_key_name
+        sorted(nodes, key=get_node_key_name),
+        get_node_key_name,
     ):
-        grouped_nodes = list(grouped_nodes)
+        grouped_nodes = list(grouped_nodes)  # noqa: PLW2901
 
         # The merged node inherits its attributes from the final node in the group.
         (last_node_key, last_node_value) = grouped_nodes[-1]
-        value_types = set(type(value) for (_, value) in grouped_nodes)
+        value_types = {type(value) for (_, value) in grouped_nodes}
 
         if len(value_types) > 1:
             raise ValueError(
-                f'Incompatible types found when trying to merge "{key_name}:" values across configuration files: {", ".join(value_type.id for value_type in value_types)}'
+                f'Incompatible types found when trying to merge "{key_name}:" values across configuration files: {", ".join(value_type.id for value_type in value_types)}',
             )
 
         # If we're dealing with MappingNodes, recurse and merge its values as well.
@@ -351,7 +356,7 @@ def deep_merge_nodes(nodes):
                             comment=last_node_value.comment,
                             anchor=last_node_value.anchor,
                         ),
-                    )
+                    ),
                 )
 
             continue
@@ -374,7 +379,7 @@ def deep_merge_nodes(nodes):
                             comment=last_node_value.comment,
                             anchor=last_node_value.anchor,
                         ),
-                    )
+                    ),
                 )
 
             continue

borgmatic/config/normalize.py

@@ -25,12 +25,12 @@ def normalize_sections(config_filename, config):
         and location.get('prefix') != consistency.get('prefix')
     ):
         raise ValueError(
-            'The retention prefix and the consistency prefix cannot have different values (unless one is not set).'
+            'The retention prefix and the consistency prefix cannot have different values (unless one is not set).',
         )
 
     if storage.get('umask') and hooks.get('umask') and storage.get('umask') != hooks.get('umask'):
         raise ValueError(
-            'The storage umask and the hooks umask cannot have different values (unless one is not set).'
+            'The storage umask and the hooks umask cannot have different values (unless one is not set).',
         )
 
     any_section_upgraded = False
@@ -51,8 +51,8 @@ def normalize_sections(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: Configuration sections (like location:, storage:, retention:, consistency:, and hooks:) are deprecated and support will be removed from a future release. To prepare for this, move your options out of sections to the global scope.',
-                )
-            )
+                ),
+            ),
         ]
 
     return []
@@ -68,7 +68,7 @@ def make_command_hook_deprecation_log(config_filename, option_name):  # pragma:
             levelno=logging.WARNING,
             levelname='WARNING',
             msg=f'{config_filename}: {option_name} is deprecated and support will be removed from a future release. Use commands: instead.',
-        )
+        ),
     )
 
 
@@ -90,7 +90,7 @@ def normalize_commands(config_filename, config):
                 {
                     preposition: 'repository',
                     'run': commands,
-                }
+                },
             )
 
     # Normalize "before_backup", "before_prune", "after_backup", "after_prune", etc.
@@ -108,7 +108,7 @@ def normalize_commands(config_filename, config):
                     preposition: 'action',
                     'when': [action_name],
                     'run': commands,
-                }
+                },
             )
 
     # Normalize "on_error".
@@ -121,7 +121,7 @@ def normalize_commands(config_filename, config):
                 'after': 'error',
                 'when': ['create', 'prune', 'compact', 'check'],
                 'run': commands,
-            }
+            },
         )
 
     # Normalize "before_everything" and "after_everything".
@@ -136,13 +136,13 @@ def normalize_commands(config_filename, config):
                     preposition: 'everything',
                     'when': ['create'],
                     'run': commands,
-                }
+                },
             )
 
     return logs
 
 
-def normalize(config_filename, config):
+def normalize(config_filename, config):  # noqa: PLR0912, PLR0915
     '''
     Given a configuration filename and a configuration dict of its loaded contents, apply particular
     hard-coded rules to normalize the configuration to adhere to the current schema. Return any log
@@ -160,8 +160,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The borgmatic_source_directory option is deprecated and will be removed from a future release. Use user_runtime_directory and user_state_directory instead.',
-                )
-            )
+                ),
+            ),
         )
 
     # Upgrade exclude_if_present from a string to a list.
@@ -173,8 +173,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The exclude_if_present option now expects a list value. String values for this option are deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['exclude_if_present'] = [exclude_if_present]
 
@@ -191,8 +191,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The store_config_files option has moved under the bootstrap hook. Specifying store_config_files at the global scope is deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         del config['store_config_files']
         config['bootstrap']['store_config_files'] = store_config_files
@@ -206,8 +206,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects a key/value pair with "ping_url" as a key. String values for this option are deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['healthchecks'] = {'ping_url': healthchecks}
 
@@ -219,8 +219,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['cronitor'] = {'ping_url': cronitor}
 
@@ -232,8 +232,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['pagerduty'] = {'integration_key': pagerduty}
 
@@ -245,8 +245,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['cronhub'] = {'ping_url': cronhub}
 
@@ -259,8 +259,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The checks option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['checks'] = [{'name': check_type} for check_type in checks]
 
@@ -273,8 +273,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The numeric_owner option has been renamed to numeric_ids. numeric_owner is deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['numeric_ids'] = numeric_owner
 
@@ -286,8 +286,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The bsd_flags option has been renamed to flags. bsd_flags is deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['flags'] = bsd_flags
 
@@ -299,8 +299,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The remote_rate_limit option has been renamed to upload_rate_limit. remote_rate_limit is deprecated and support will be removed from a future release.',
-                )
-            )
+                ),
+            ),
         )
         config['upload_rate_limit'] = remote_rate_limit
 
@@ -314,8 +314,8 @@ def normalize(config_filename, config):
                         levelno=logging.WARNING,
                         levelname='WARNING',
                         msg=f'{config_filename}: The repositories option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.',
-                    )
-                )
+                    ),
+                ),
             )
             config['repositories'] = [
                 {'path': repository} if isinstance(repository, str) else repository
@@ -338,27 +338,22 @@ def normalize(config_filename, config):
                             levelno=logging.WARNING,
                             levelname='WARNING',
                             msg=f'{config_filename}: Repository paths containing "~" are deprecated in borgmatic and support will be removed from a future release.',
-                        )
-                    )
+                        ),
+                    ),
                 )
+
             if ':' in repository_path:
                 if repository_path.startswith('file://'):
                     updated_repository_path = os.path.abspath(
-                        repository_path.partition('file://')[-1]
+                        repository_path.partition('file://')[-1],
                     )
                     config['repositories'].append(
                         dict(
                             repository_dict,
                             path=updated_repository_path,
-                        )
+                        ),
                     )
-                elif (
-                    repository_path.startswith('ssh://')
-                    or repository_path.startswith('sftp://')
-                    or repository_path.startswith('rclone:')
-                    or repository_path.startswith('s3:')
-                    or repository_path.startswith('b2:')
-                ):
+                elif repository_path.startswith(('ssh://', 'sftp://', 'rclone:', 's3:', 'b2:')):
                     config['repositories'].append(repository_dict)
                 else:
                     rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
@@ -368,14 +363,14 @@ def normalize(config_filename, config):
                                 levelno=logging.WARNING,
                                 levelname='WARNING',
                                 msg=f'{config_filename}: Remote repository paths without ssh://, sftp://, rclone:, s3:, or b2:, syntax are deprecated and support will be removed from a future release. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
-                            )
-                        )
+                            ),
+                        ),
                     )
                     config['repositories'].append(
                         dict(
                             repository_dict,
                             path=rewritten_repository_path,
-                        )
+                        ),
                     )
             else:
                 config['repositories'].append(repository_dict)
@@ -387,8 +382,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The prefix option is deprecated and support will be removed from a future release. Use archive_name_format or match_archives instead.',
-                )
-            )
+                ),
+            ),
         )
 
     return logs

borgmatic/config/override.py

@@ -18,7 +18,7 @@ def set_values(config, keys, value):
     if len(keys) == 1:
         if isinstance(config, list):
             raise ValueError(
-                'When overriding a list option, the value must use list syntax (e.g., "[foo, bar]" or "[{key: value}]" as appropriate)'
+                'When overriding a list option, the value must use list syntax (e.g., "[foo, bar]" or "[{key: value}]" as appropriate)',
             )
 
         config[first_key] = value
@@ -69,11 +69,11 @@ def type_for_option(schema, option_keys):
     '''
     option_schema = schema
 
-    for key in option_keys:
-        try:
+    try:
+        for key in option_keys:
             option_schema = option_schema['properties'][key]
-        except KeyError:
-            return None
+    except KeyError:
+        return None
 
     try:
         return option_schema['type']
@@ -103,8 +103,8 @@ def parse_overrides(raw_overrides, schema):
 
     parsed_overrides = []
 
-    for raw_override in raw_overrides:
-        try:
+    try:
+        for raw_override in raw_overrides:
             raw_keys, value = raw_override.split('=', 1)
             keys = tuple(raw_keys.split('.'))
             option_type = type_for_option(schema, keys)
@@ -113,14 +113,14 @@ def parse_overrides(raw_overrides, schema):
                 (
                     keys,
                     convert_value_type(value, option_type),
-                )
+                ),
             )
-        except ValueError:
-            raise ValueError(
-                f"Invalid override '{raw_override}'. Make sure you use the form: OPTION=VALUE or OPTION.SUBOPTION=VALUE"
-            )
-        except ruamel.yaml.error.YAMLError as error:
-            raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
+    except ValueError:
+        raise ValueError(
+            f"Invalid override '{raw_override}'. Make sure you use the form: OPTION=VALUE or OPTION.SUBOPTION=VALUE",
+        )
+    except ruamel.yaml.error.YAMLError as error:
+        raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
 
     return tuple(parsed_overrides)
 
@@ -139,7 +139,7 @@ def apply_overrides(config, schema, raw_overrides):
 
     if overrides:
         logger.warning(
-            "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set."
+            "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set.",
         )
 
     for keys, value in overrides:

borgmatic/config/paths.py

@@ -1,10 +1,20 @@
+import contextlib
 import logging
 import os
 import tempfile
+from enum import Enum
 
 logger = logging.getLogger(__name__)
 
 
+class Systemd_directories(Enum):
+    RUNTIME_DIRECTORY = 0
+    STATE_DIRECTORY = 1
+    CACHE_DIRECTORY = 2
+    LOGS_DIRECTORY = 3
+    CONFIGURATION_DIRECTORY = 4
+
+
 def expand_user_in_path(path):
     '''
     Given a directory path, expand any tildes in it.
@@ -15,6 +25,17 @@ def expand_user_in_path(path):
         return None
 
 
+def resolve_systemd_directory(directory):
+    '''
+    Given a systemd directory environment variable enum, read the value if set and return the first
+    configured directory.
+    '''
+    separator = ':'
+
+    paths = os.environ.get(directory.name)
+    return paths.split(separator)[0] if paths else None
+
+
 def get_working_directory(config):  # pragma: no cover
     '''
     Given a configuration dict, get the working directory from it, expanding any tildes.
@@ -34,7 +55,8 @@ TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-'
 
 
 def replace_temporary_subdirectory_with_glob(
-    path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX
+    path,
+    temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX,
 ):
     '''
     Given an absolute temporary directory path and an optional temporary directory prefix, look for
@@ -94,7 +116,9 @@ class Runtime_directory:
         runtime_directory = (
             config.get('user_runtime_directory')
             or os.environ.get('XDG_RUNTIME_DIR')  # Set by PAM on Linux.
-            or os.environ.get('RUNTIME_DIRECTORY')  # Set by systemd if configured.
+            or resolve_systemd_directory(
+                Systemd_directories.RUNTIME_DIRECTORY
+            )  # Set by systemd if configured.
         )
 
         if runtime_directory:
@@ -103,7 +127,9 @@ class Runtime_directory:
 
             self.temporary_directory = None
         else:
-            base_directory = os.environ.get('TMPDIR') or os.environ.get('TEMP') or '/tmp'
+            base_directory = (
+                os.environ.get('TMPDIR') or os.environ.get('TEMP') or '/tmp'  # noqa: S108
+            )
 
             if not base_directory.startswith(os.path.sep):
                 raise ValueError('The temporary directory must be an absolute path')
@@ -122,7 +148,7 @@ class Runtime_directory:
                 base_path if final_directory == 'borgmatic' else runtime_directory,
                 '.',  # Borg 1.4+ "slashdot" hack.
                 'borgmatic',
-            )
+            ),
         )
         os.makedirs(self.runtime_path, mode=0o700, exist_ok=True)
 
@@ -139,13 +165,11 @@ class Runtime_directory:
         Delete any temporary directory that was created as part of initialization.
         '''
         if self.temporary_directory:
-            try:
-                self.temporary_directory.cleanup()
             # The cleanup() call errors if, for instance, there's still a
             # mounted filesystem within the temporary directory. There's
             # nothing we can do about that here, so swallow the error.
-            except OSError:
-                pass
+            with contextlib.suppress(OSError):
+                self.temporary_directory.cleanup()
 
 
 def make_runtime_directory_glob(borgmatic_runtime_directory):
@@ -158,7 +182,7 @@ def make_runtime_directory_glob(borgmatic_runtime_directory):
         *(
             '*' if subdirectory.startswith(TEMPORARY_DIRECTORY_PREFIX) else subdirectory
             for subdirectory in os.path.normpath(borgmatic_runtime_directory).split(os.path.sep)
-        )
+        ),
     )
 
 
@@ -172,8 +196,10 @@ def get_borgmatic_state_directory(config):
         os.path.join(
             config.get('user_state_directory')
             or os.environ.get('XDG_STATE_HOME')
-            or os.environ.get('STATE_DIRECTORY')  # Set by systemd if configured.
+            or resolve_systemd_directory(
+                Systemd_directories.STATE_DIRECTORY
+            )  # Set by systemd if configured.
             or '~/.local/state',
             'borgmatic',
-        )
+        ),
     )

borgmatic/config/schema.py

@@ -14,8 +14,8 @@ def get_properties(schema):
             item
             for item in itertools.chain(
                 *itertools.zip_longest(
-                    *[sub_schema['properties'].items() for sub_schema in schema['oneOf']]
-                )
+                    *[sub_schema['properties'].items() for sub_schema in schema['oneOf']],
+                ),
             )
             if item is not None
         )
@@ -61,12 +61,6 @@ def compare_types(schema_type, target_types, match=any):
     list must be in the target types.
     '''
     if isinstance(schema_type, list):
-        if match(element_schema_type in target_types for element_schema_type in schema_type):
-            return True
+        return match(element_schema_type in target_types for element_schema_type in schema_type)
 
-        return False
-
-    if schema_type in target_types:
-        return True
-
-    return False
+    return schema_type in target_types

borgmatic/config/schema.yaml

@@ -21,12 +21,22 @@ properties:
             type: string
         description: |
             List of source directories and files to back up. Globs and tildes
-            are expanded. Do not backslash spaces in path names.
+            are expanded. Do not backslash spaces in path names. Be aware that
+            by default, Borg treats missing source directories as warnings
+            rather than errors. If you'd like to change that behavior, see
+            https://torsion.org/borgmatic/how-to/customize-warnings-and-errors/
+            or the "source_directories_must_exist" option.
         example:
             - /home
             - /etc
             - /var/log/syslog*
             - /home/user/path with spaces
+    source_directories_must_exist:
+        type: boolean
+        description: |
+            If true, then source directories (and root pattern paths) must
+            exist. If they don't, an error is raised. Defaults to false.
+        example: true
     repositories:
         type: array
         items:
@@ -72,7 +82,8 @@ properties:
                     description: |
                         Whether any missing parent directories of the repository
                         path should be created, only used for the repo-create
-                        action. Defaults to false.
+                        action. Defaults to false. (This option is supported 
+                        for Borg 1.x only.)
                     example: true
         description: |
             A required list of local or remote repositories with paths and
@@ -162,9 +173,13 @@ properties:
             type: string
         description: |
             Any paths matching these patterns are included/excluded from
-            backups. Globs are expanded. (Tildes are not.) See the output of
-            "borg help patterns" for more details. Quote any value if it
-            contains leading punctuation, so it parses correctly.
+            backups. Recursion root patterns ("R ...") are effectively the same
+            as "source_directories"; they tell Borg which paths to backup
+            (modulo any excludes). Globs are expanded. (Tildes are not.) See
+            the output of "borg help patterns" for more details. Quote any value
+            if it contains leading punctuation, so it parses correctly. Also use
+            leading slashes in absolute paths, or data source hooks may be
+            unable to rewrite patterns as needed.
         example:
             - 'R /'
             - '- /home/*/.cache'
@@ -231,7 +246,8 @@ properties:
     exclude_nodump:
         type: boolean
         description: |
-            Exclude files with the NODUMP flag. Defaults to false.
+            Exclude files with the NODUMP flag. Defaults to false. (This option
+            is supported for Borg 1.x only.)
         example: true
     borgmatic_source_directory:
         type: string
@@ -247,7 +263,7 @@ properties:
             Path for storing temporary runtime data like streaming database
             dumps and bootstrap metadata. borgmatic automatically creates and
             uses a "borgmatic" subdirectory here. Defaults to $XDG_RUNTIME_DIR
-            or or $TMPDIR or $TEMP or /run/user/$UID.
+            or $TMPDIR or $TEMP or /run/user/$UID.
         example: /run/user/1001
     user_state_directory:
         type: string
@@ -258,12 +274,6 @@ properties:
             create the check records again (and therefore re-run checks).
             Defaults to $XDG_STATE_HOME or ~/.local/state.
         example: /var/lib/borgmatic
-    source_directories_must_exist:
-        type: boolean
-        description: |
-            If true, then source directories (and root pattern paths) must
-            exist. If they don't, an error is raised. Defaults to false.
-        example: true
     encryption_passcommand:
         type: string
         description: |
@@ -274,7 +284,7 @@ properties:
             encryption_passphrase takes precedence. This can also be used to
             access encrypted systemd service credentials. Defaults to not set.
             For more details, see:
-            https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+            https://torsion.org/borgmatic/how-to/provide-your-passwords/
         example: "secret-tool lookup borg-repository repo-name"
     encryption_passphrase:
         type: string
@@ -405,14 +415,20 @@ properties:
         type: string
         description: |
             Path for Borg security and encryption nonce files. Defaults to
-            $borg_base_directory/.config/borg/security
+            $borg_config_directory/security
         example: /path/to/base/config/security
     borg_keys_directory:
         type: string
         description: |
             Path for Borg encryption key files. Defaults to
-            $borg_base_directory/.config/borg/keys
+            $borg_config_directory/keys
         example: /path/to/base/config/keys
+    borg_key_file:
+        type: string
+        description: |
+            Path for the Borg repository key file, for use with a repository
+            created with "keyfile" encryption.
+        example: /path/to/base/config/keyfile
     borg_exit_codes:
         type: array
         items:
@@ -478,6 +494,27 @@ properties:
             If match_archives is not specified, borgmatic defaults to deriving
             the match_archives value from archive_name_format.
         example: "sh:{hostname}-*"
+    file_list_format:
+        type: string
+        description: |
+            Borg format for the files listing of the "list" action. Corresponds
+            to the "--format" flag. Defaults to "{mode} {user:6} {group:6}
+            {size:8} {mtime} {path}{extra}{NL}". With "--json", the form of the
+            format is ignored, but the keys used in it are added to the JSON
+            output. See
+            https://borgbackup.readthedocs.io/en/stable/usage/list.html for
+            details.
+        example: "{path}{extra}{NL}"
+    archive_list_format:
+        type: string
+        description: |
+            Borg format for the archives listing of the "repo-list" action.
+            Corresponds to the "--format" flag. Defaults to "{archive:<36}
+            {time} [{id}]{NL}". With "--json", the form of the format is
+            ignored, but the keys used in it are added to the JSON output. See
+            https://borgbackup.readthedocs.io/en/stable/usage/list.html for
+            details.
+        example: "{archive}{NL}"
     relocated_repo_access_is_ok:
         type: boolean
         description: |
@@ -490,6 +527,19 @@ properties:
             Bypass Borg error about a previously unknown unencrypted repository.
             Defaults to false.
         example: true
+    debug_passphrase:
+        type: boolean
+        description: |
+            When set true, display debugging information that includes 
+            passphrases used and passphrase related environment variables set. 
+            Defaults to false.
+        example: true
+    display_passphrase:
+        type: boolean
+        description: |
+            When set true, always shows passphrase and its hex UTF-8 byte
+            sequence. Defaults to false.
+        example: true
     check_i_know_what_i_am_doing:
         type: boolean
         description: |
@@ -500,30 +550,123 @@ properties:
         type: object
         additionalProperties: false
         properties:
-            init:
+            break_lock:
                 type: string
                 description: |
-                  Extra command-line options to pass to "borg init".
+                  Extra command-line options to pass to "borg break-lock".
                 example: "--extra-option"
-            create:
+            check:
                 type: string
                 description: |
-                  Extra command-line options to pass to "borg create".
-                example: "--extra-option"
-            prune:
-                type: string
-                description: |
-                  Extra command-line options to pass to "borg prune".
+                  Extra command-line options to pass to "borg check".
                 example: "--extra-option"
             compact:
                 type: string
                 description: |
                   Extra command-line options to pass to "borg compact".
                 example: "--extra-option"
-            check:
+            create:
                 type: string
                 description: |
-                  Extra command-line options to pass to "borg check".
+                  Extra command-line options to pass to "borg create".
+                example: "--extra-option"
+            delete:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg delete".
+                example: "--extra-option"
+            export_tar:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg export-tar".
+                example: "--extra-option"
+            extract:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg extract".
+                example: "--extra-option"
+            key_export:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg key export".
+                example: "--extra-option"
+            key_import:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg key import".
+                example: "--extra-option"
+            key_change_passphrase:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg key
+                  change-passphrase".
+                example: "--extra-option"
+            info:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg info".
+                example: "--extra-option"
+            init:
+                type: string
+                description: |
+                  Deprecated. Use "repo_create" instead. Extra command-line
+                  options to pass to "borg init" / "borg repo-create".
+                example: "--extra-option"
+            list:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg list".
+                example: "--extra-option"
+            mount:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg mount".
+                example: "--extra-option"
+            prune:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg prune".
+                example: "--extra-option"
+            recreate:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg recreate".
+                example: "--extra-option"
+            rename:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg rename".
+                example: "--extra-option"
+            repo_create:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg init" / "borg
+                  repo-create".
+                example: "--extra-option"
+            repo_delete:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg repo-delete".
+                example: "--extra-option"
+            repo_info:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg repo-info".
+                example: "--extra-option"
+            repo_list:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg repo-list".
+                example: "--extra-option"
+            transfer:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg transfer".
+                example: "--extra-option"
+            umount:
+                type: string
+                description: |
+                  Extra command-line options to pass to "borg umount".
                 example: "--extra-option"
         description: |
             Additional options to pass directly to particular Borg commands,
@@ -878,6 +1021,8 @@ properties:
         type: string
         description: |
             Python format string used for log messages written to the log file.
+            See https://docs.python.org/3/library/logging.html (and specifically
+            the LogRecord attributes with "{}-formatting") for details.
         example: "[{asctime}] {levelname}: {prefix}{message}"
     monitoring_verbosity:
         type: integer
@@ -1309,6 +1454,23 @@ properties:
                         implicitly enables read_special (see above) to support
                         dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1510,6 +1672,32 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                skip_names:
+                    type: array
+                    items:
+                        type: string
+                    description: |
+                        Database names to skip when dumping "all" databases.
+                        Ignored when the database name is not "all".
+                    example:
+                        - cache
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1531,6 +1719,20 @@ properties:
                     description: |
                         Port to restore to. Defaults to the "port" option.
                     example: 5433
+                socket_path:
+                    type: string
+                    description: |
+                        Path of a Unix socket to connect to instead of a remote
+                        hostname. Ignored when "hostname" is set and not
+                        "localhost". Defaults to "/run/mysqld/mysqld.sock".
+                    example: database.example.org
+                restore_socket_path:
+                    type: string
+                    description: |
+                        Path of a Unix socket to connect to instead of a remote
+                        hostname. Ignored when "hostname" is set and not
+                        "localhost". Defaults to the "socket_path" option.
+                    example: database.example.org
                 username:
                     type: string
                     description: |
@@ -1594,18 +1796,21 @@ properties:
                 mariadb_dump_command:
                     type: string
                     description: |
-                        Command to use instead of "mariadb-dump". This can be
-                        used to run a specific mariadb_dump version (e.g., one
-                        inside a running container). If you run it from within a
-                        container, make sure to mount the path in the
-                        "user_runtime_directory" option from the host into the
-                        container at the same location. Defaults to
-                        "mariadb-dump".
+                        Command to use instead of "mariadb-dump". But set any
+                        additional command-line flags in "options", not here.
+                        This command can be used to run a specific mariadb_dump
+                        version (e.g., one inside a running container). If you
+                        run it from within a container, make sure to mount the
+                        path in the "user_runtime_directory" option from the
+                        host into the container at the same location. Defaults
+                        to "mariadb-dump".
                     example: docker exec mariadb_container mariadb-dump
                 mariadb_command:
                     type: string
                     description: |
-                        Command to run instead of "mariadb". This can be used to
+                        Command to run instead of "mariadb". But set any
+                        additional command-line flags in "list_options" or
+                        "restore_options", not here. This command can be used to
                         run a specific mariadb version (e.g., one inside a
                         running container). Defaults to "mariadb".
                     example: docker exec mariadb_container mariadb           
@@ -1675,6 +1880,32 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                skip_names:
+                    type: array
+                    items:
+                        type: string
+                    description: |
+                        Database names to skip when dumping "all" databases.
+                        Ignored when the database name is not "all".
+                    example:
+                        - cache
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1696,6 +1927,20 @@ properties:
                     description: |
                         Port to restore to. Defaults to the "port" option.
                     example: 5433
+                socket_path:
+                    type: string
+                    description: |
+                        Path of a Unix socket to connect to instead of a remote
+                        hostname. Ignored when "hostname" is set and not
+                        "localhost". Defaults to "/run/mysqld/mysqld.sock".
+                    example: database.example.org
+                restore_socket_path:
+                    type: string
+                    description: |
+                        Path of a Unix socket to connect to instead of a remote
+                        hostname. Ignored when "hostname" is set and not
+                        "localhost". Defaults to the "socket_path" option.
+                    example: database.example.org
                 username:
                     type: string
                     description: |
@@ -1759,17 +2004,21 @@ properties:
                 mysql_dump_command:
                     type: string
                     description: |
-                        Command to use instead of "mysqldump". This can be used
-                        to run a specific mysql_dump version (e.g., one inside a
-                        running container). If you run it from within a
-                        container, make sure to mount the path in the
-                        "user_runtime_directory" option from the host into the
-                        container at the same location. Defaults to "mysqldump".
+                        Command to use instead of "mysqldump". But set any
+                        additional command-line flags in "options", not here.
+                        This command can be used to run a specific mysql_dump
+                        version (e.g., one inside a running container). If you
+                        run it from within a container, make sure to mount the
+                        path in the "user_runtime_directory" option from the
+                        host into the container at the same location. Defaults
+                        to "mysqldump".
                     example: docker exec mysql_container mysqldump
                 mysql_command:
                     type: string
                     description: |
-                        Command to run instead of "mysql". This can be used to
+                        Command to run instead of "mysql". But set any
+                        additional command-line flags in "list_options" or
+                        "restore_options", not here. This command can be used to
                         run a specific mysql version (e.g., one inside a running
                         container). Defaults to "mysql".
                     example: docker exec mysql_container mysql
@@ -1848,6 +2097,11 @@ properties:
                         read_special (see above) to support dump and restore
                         streaming.
                     example: /var/lib/sqlite/users.db
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
                 restore_path:
                     type: string
                     description: |
@@ -1896,6 +2150,23 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -2052,17 +2323,17 @@ properties:
                         type: string
                         description: |
                             The message body to publish.
-                        example: Your backups have failed.
+                        example: Your backups have started.
                     priority:
                         type: string
                         description: |
                             The priority to set.
-                        example: urgent
+                        example: min
                     tags:
                         type: string
                         description: |
                             Tags to attach to the message.
-                        example: incoming_envelope
+                        example: borgmatic
             finish:
                 type: object
                 additionalProperties: false
@@ -2076,17 +2347,17 @@ properties:
                         type: string
                         description: |
                             The message body to publish.
-                        example: Your backups have failed.
+                        example: Your backups have finished.
                     priority:
                         type: string
                         description: |
                             The priority to set.
-                        example: urgent
+                        example: min
                     tags:
                         type: string
                         description: |
                             Tags to attach to the message.
-                        example: incoming_envelope
+                        example: borgmatic,+1
             fail:
                 type: object
                 additionalProperties: false
@@ -2105,12 +2376,12 @@ properties:
                         type: string
                         description: |
                             The priority to set.
-                        example: urgent
+                        example: max
                     tags:
                         type: string
                         description: |
                             Tags to attach to the message.
-                        example: incoming_envelope
+                        example: borgmatic,-1,skull
             states:
                 type: array
                 items:
@@ -2811,6 +3082,12 @@ properties:
                     project monitor. Used along with the data source name URL to
                     construct a cron URL.
                 example: mymonitor
+            environment:
+                type: string
+                description: |
+                    Sentry monitor environment used in the call to Sentry. If
+                    not set, the Sentry default is used.
+                example: production
             states:
                 type: array
                 items:
@@ -2864,7 +3141,8 @@ properties:
             findmnt_command:
                 type: string
                 description: |
-                    Command to use instead of "findmnt".
+                    Deprecated and unused. Was the command to use instead of
+                    "findmnt".
                 example: /usr/local/bin/findmnt
         description: |
             Configuration for integration with the Btrfs filesystem.
@@ -2915,6 +3193,25 @@ properties:
         description: |
             Configuration for integration with Linux LVM (Logical Volume
             Manager).
+    systemd:
+        type: object
+        additionalProperties: false
+        properties:
+            systemd_creds_command:
+                type: string
+                description: |
+                    Command to use instead of "systemd-creds". Only used as a
+                    fallback when borgmatic is run outside of a systemd service.
+                example: /usr/local/bin/systemd-creds
+            encrypted_credentials_directory:
+                type: string
+                description: |
+                    Directory containing encrypted credentials for
+                    "systemd-creds" to use instead of
+                    "/etc/credstore.encrypted".
+                example: /path/to/credstore.encrypted
+        description: |
+            Configuration for integration with systemd credentials.
     container:
         type: object
         additionalProperties: false
@@ -2935,6 +3232,14 @@ properties:
                 description: |
                     Command to use instead of "keepassxc-cli".
                 example: /usr/local/bin/keepassxc-cli
+            ask_for_password:
+                type: boolean
+                description: |
+                    Whether keepassxc-cli should prompt the user for a password.
+                    Disabling this is only really useful if you're unlocking
+                    your KeePassXC database with a key file instead of a
+                    password. Defaults to true.
+                example: false
             key_file:
                 type: string
                 description: |

borgmatic/config/validate.py

@@ -17,7 +17,7 @@ def schema_filename():
     '''
     schema_path = os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml')
 
-    with open(schema_path):
+    with open(schema_path, encoding='utf-8'):
         return schema_path
 
 
@@ -97,7 +97,11 @@ def apply_logical_validation(config_filename, parsed_configuration):
 
 
 def parse_configuration(
-    config_filename, schema_filename, arguments, overrides=None, resolve_env=True
+    config_filename,
+    schema_filename,
+    arguments,
+    overrides=None,
+    resolve_env=True,
 ):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
@@ -147,7 +151,8 @@ def parse_configuration(
 
     if validation_errors:
         raise Validation_error(
-            config_filename, tuple(format_json_error(error) for error in validation_errors)
+            config_filename,
+            tuple(format_json_error(error) for error in validation_errors),
         )
 
     apply_logical_validation(config_filename, config)
@@ -166,13 +171,14 @@ def normalize_repository_path(repository, base=None):
         return (
             os.path.abspath(os.path.join(base, repository)) if base else os.path.abspath(repository)
         )
-    elif repository.startswith('file://'):
+
+    if repository.startswith('file://'):
         local_path = repository.partition('file://')[-1]
         return (
             os.path.abspath(os.path.join(base, local_path)) if base else os.path.abspath(local_path)
         )
-    else:
-        return repository
+
+    return repository
 
 
 def glob_match(first, second):
@@ -194,11 +200,13 @@ def repositories_match(first, second):
     '''
     if isinstance(first, str):
         first = {'path': first, 'label': first}
+
     if isinstance(second, str):
         second = {'path': second, 'label': second}
 
     return glob_match(first.get('label'), second.get('label')) or glob_match(
-        normalize_repository_path(first.get('path')), normalize_repository_path(second.get('path'))
+        normalize_repository_path(first.get('path')),
+        normalize_repository_path(second.get('path')),
     )
 
 
@@ -219,7 +227,7 @@ def guard_configuration_contains_repository(repository, configurations):
             for config in configurations.values()
             for config_repository in config['repositories']
             if repositories_match(config_repository, repository)
-        )
+        ),
     )
 
     if count == 0:

borgmatic/execute.py

@@ -31,6 +31,7 @@ def interpret_exit_code(command, exit_code, borg_local_path=None, borg_exit_code
     '''
     if exit_code is None:
         return Exit_status.STILL_RUNNING
+
     if exit_code == 0:
         return Exit_status.SUCCESS
 
@@ -42,12 +43,13 @@ def interpret_exit_code(command, exit_code, borg_local_path=None, borg_exit_code
 
                 if treat_as == 'error':
                     logger.error(
-                        f'Treating exit code {exit_code} as an error, as per configuration'
+                        f'Treating exit code {exit_code} as an error, as per configuration',
                     )
                     return Exit_status.ERROR
-                elif treat_as == 'warning':
+
+                if treat_as == 'warning':
                     logger.warning(
-                        f'Treating exit code {exit_code} as a warning, as per configuration'
+                        f'Treating exit code {exit_code} as a warning, as per configuration',
                     )
                     return Exit_status.WARNING
 
@@ -102,7 +104,7 @@ def append_last_lines(last_lines, captured_output, line, output_log_level):
         logger.log(output_log_level, line)
 
 
-def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes):
+def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes):  # noqa: PLR0912
     '''
     Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
     process with the requested log level. Additionally, raise a CalledProcessError if a process
@@ -131,7 +133,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
     still_running = True
 
     # Log output for each process until they all exit.
-    while True:
+    while True:  # noqa: PLR1702
         if output_buffers:
             (ready_buffers, _, _) = select.select(output_buffers, [], [])
 
@@ -156,8 +158,8 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
                     if not line or not ready_process:
                         break
 
-                    # Keep the last few lines of output in case the process errors, and we need the output for
-                    # the exception below.
+                    # Keep the last few lines of output in case the process errors, and we need the
+                    # output for the exception below.
                     append_last_lines(
                         buffer_last_lines[ready_buffer],
                         captured_outputs[ready_process],
@@ -181,7 +183,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
             command = process.args.split(' ') if isinstance(process.args, str) else process.args
             exit_status = interpret_exit_code(command, exit_code, borg_local_path, borg_exit_codes)
 
-            if exit_status in (Exit_status.ERROR, Exit_status.WARNING):
+            if exit_status in {Exit_status.ERROR, Exit_status.WARNING}:
                 # If an error occurs, include its output in the raised exception so that we don't
                 # inadvertently hide error output.
                 output_buffer = output_buffer_for_process(process, exclude_stdouts)
@@ -194,7 +196,10 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
                         break
 
                     append_last_lines(
-                        last_lines, captured_outputs[process], line, output_log_level=logging.ERROR
+                        last_lines,
+                        captured_outputs[process],
+                        line,
+                        output_log_level,
                     )
 
                 if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
@@ -209,7 +214,9 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
 
                 if exit_status == Exit_status.ERROR:
                     raise subprocess.CalledProcessError(
-                        exit_code, command_for_process(process), '\n'.join(last_lines)
+                        exit_code,
+                        command_for_process(process),
+                        '\n'.join(last_lines),
                     )
 
                 still_running = False
@@ -220,6 +227,8 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
             process: '\n'.join(output_lines) for process, output_lines in captured_outputs.items()
         }
 
+    return None
+
 
 SECRET_COMMAND_FLAG_NAMES = {'--password'}
 
@@ -255,19 +264,19 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non
             ' '.join(
                 tuple(
                     f'{key}=***'
-                    for key in (environment or {}).keys()
+                    for key in (environment or {})
                     if any(
                         key.startswith(prefix)
                         for prefix in PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG
                     )
                 )
-                + mask_command_secrets(full_command)
+                + mask_command_secrets(full_command),
             ),
             width=MAX_LOGGED_COMMAND_LENGTH,
             placeholder=' ...',
         )
         + (f" < {getattr(input_file, 'name', input_file)}" if input_file else '')
-        + (f" > {getattr(output_file, 'name', output_file)}" if output_file else '')
+        + (f" > {getattr(output_file, 'name', output_file)}" if output_file else ''),
     )
 
 
@@ -308,7 +317,7 @@ def execute_command(
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
-    process = subprocess.Popen(
+    process = subprocess.Popen(  # noqa: S603
         command,
         stdin=input_file,
         stdout=None if do_not_capture else (output_file or subprocess.PIPE),
@@ -330,6 +339,8 @@ def execute_command(
             borg_exit_codes,
         )
 
+    return None
+
 
 def execute_command_and_capture_output(
     full_command,
@@ -359,7 +370,7 @@ def execute_command_and_capture_output(
     command = ' '.join(full_command) if shell else full_command
 
     try:
-        output = subprocess.check_output(
+        output = subprocess.check_output(  # noqa: S603
             command,
             stdin=input_file,
             stderr=subprocess.STDOUT if capture_stderr else None,
@@ -374,6 +385,7 @@ def execute_command_and_capture_output(
             == Exit_status.ERROR
         ):
             raise
+
         output = error.output
 
     return output.decode() if output is not None else None
@@ -416,7 +428,7 @@ def execute_command_with_processes(
     command = ' '.join(full_command) if shell else full_command
 
     try:
-        command_process = subprocess.Popen(
+        command_process = subprocess.Popen(  # noqa: S603
             command,
             stdin=input_file,
             stdout=None if do_not_capture else (output_file or subprocess.PIPE),
@@ -435,11 +447,12 @@ def execute_command_with_processes(
             if process.poll() is None:
                 process.stdout.read(0)
                 process.kill()
+
         raise
 
     with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
         captured_outputs = log_outputs(
-            tuple(processes) + (command_process,),
+            (*processes, command_process),
             (input_file, output_file),
             output_log_level,
             borg_local_path,
@@ -448,3 +461,5 @@ def execute_command_with_processes(
 
     if output_log_level is None:
         return captured_outputs.get(command_process)
+
+    return None

borgmatic/hooks/command.py

@@ -42,7 +42,7 @@ def interpolate_context(hook_description, command, context):
         # be a Borg placeholder, as Borg should hopefully consume it.
         if unsupported_variable not in BORG_PLACEHOLDERS:
             logger.warning(
-                f'Variable "{unsupported_variable}" is not supported in the {hook_description} hook'
+                f'Variable "{unsupported_variable}" is not supported in the {hook_description} hook',
             )
 
     return command
@@ -86,7 +86,7 @@ def filter_hooks(command_hooks, before=None, after=None, action_names=None, stat
     )
 
 
-def execute_hooks(command_hooks, umask, working_directory, dry_run, **context):
+def execute_hooks(command_hooks, umask, working_directory, dry_run, **context):  # noqa: PLR0912
     '''
     Given a sequence of command hook dicts from configuration, a umask to execute with (or None), a
     working directory to execute with, and whether this is a dry run, run the commands for each
@@ -139,7 +139,7 @@ def execute_hooks(command_hooks, umask, working_directory, dry_run, **context):
                 if dry_run:
                     continue
 
-                borgmatic.execute.execute_command(
+                borgmatic.execute.execute_command(  # noqa: S604
                     [command],
                     output_log_level=(
                         logging.ERROR if hook_config.get('after') == 'error' else logging.ANSWER

borgmatic/hooks/credential/container.py

@@ -34,7 +34,8 @@ def load_credential(hook_config, config, credential_parameters):
                 config.get('working_directory', ''),
                 (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY),
                 secret_name,
-            )
+            ),
+            encoding='utf-8',
         ) as secret_file:
             return secret_file.read().rstrip(os.linesep)
     except (FileNotFoundError, OSError) as error:

borgmatic/hooks/credential/file.py

@@ -23,7 +23,8 @@ def load_credential(hook_config, config, credential_parameters):
 
     try:
         with open(
-            os.path.join(config.get('working_directory', ''), expanded_credential_path)
+            os.path.join(config.get('working_directory', ''), expanded_credential_path),
+            encoding='utf-8',
         ) as credential_file:
             return credential_file.read().rstrip(os.linesep)
     except (FileNotFoundError, OSError) as error:

borgmatic/hooks/credential/keepassxc.py

@@ -29,6 +29,7 @@ def load_credential(hook_config, config, credential_parameters):
     command = (
         tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli')))
         + ('show', '--show-protected', '--attributes', 'Password')
+        + (('--no-password',) if not (hook_config or {}).get('ask_for_password', True) else ())
         + (
             ('--key-file', hook_config['key_file'])
             if hook_config and hook_config.get('key_file')

borgmatic/hooks/credential/parse.py

@@ -120,5 +120,8 @@ def resolve_credential(value, config):
         raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
 
     return borgmatic.hooks.dispatch.call_hook(
-        'load_credential', config, hook_name, tuple(credential_parameters)
+        'load_credential',
+        config,
+        hook_name,
+        tuple(credential_parameters),
     )

borgmatic/hooks/credential/systemd.py

@@ -1,6 +1,9 @@
 import logging
 import os
 import re
+import shlex
+
+import borgmatic.execute
 
 logger = logging.getLogger(__name__)
 
@@ -24,18 +27,33 @@ def load_credential(hook_config, config, credential_parameters):
 
         raise ValueError(f'Cannot load invalid credential name: "{name}"')
 
-    credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
-
-    if not credentials_directory:
-        raise ValueError(
-            f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
-        )
-
     if not CREDENTIAL_NAME_PATTERN.match(credential_name):
         raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
 
+    credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
+
+    if not credentials_directory:
+        logger.debug(
+            f'Falling back to loading credential "{credential_name}" via systemd-creds because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
+        )
+
+        command = (
+            *shlex.split((hook_config or {}).get('systemd_creds_command', 'systemd-creds')),
+            'decrypt',
+            os.path.join(
+                (hook_config or {}).get(
+                    'encrypted_credentials_directory', '/etc/credstore.encrypted'
+                ),
+                credential_name,
+            ),
+        )
+
+        return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
+
     try:
-        with open(os.path.join(credentials_directory, credential_name)) as credential_file:
+        with open(
+            os.path.join(credentials_directory, credential_name), encoding='utf-8'
+        ) as credential_file:
             return credential_file.read().rstrip(os.linesep)
     except (FileNotFoundError, OSError) as error:
         logger.warning(error)

borgmatic/hooks/data_source/bootstrap.py

@@ -1,3 +1,4 @@
+import contextlib
 import glob
 import importlib
 import json
@@ -6,6 +7,7 @@ import os
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.data_source.config
 
 logger = logging.getLogger(__name__)
 
@@ -38,7 +40,9 @@ def dump_data_sources(
         return []
 
     borgmatic_manifest_path = os.path.join(
-        borgmatic_runtime_directory, 'bootstrap', 'manifest.json'
+        borgmatic_runtime_directory,
+        'bootstrap',
+        'manifest.json',
     )
 
     if dry_run:
@@ -46,7 +50,7 @@ def dump_data_sources(
 
     os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
 
-    with open(borgmatic_manifest_path, 'w') as manifest_file:
+    with open(borgmatic_manifest_path, 'w', encoding='utf-8') as manifest_file:
         json.dump(
             {
                 'borgmatic_version': importlib.metadata.version('borgmatic'),
@@ -55,27 +59,31 @@ def dump_data_sources(
             manifest_file,
         )
 
-    patterns.extend(
-        borgmatic.borg.pattern.Pattern(
-            config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK
-        )
-        for config_path in config_paths
-    )
-    patterns.append(
+    borgmatic.hooks.data_source.config.inject_pattern(
+        patterns,
         borgmatic.borg.pattern.Pattern(
             os.path.join(borgmatic_runtime_directory, 'bootstrap'),
             source=borgmatic.borg.pattern.Pattern_source.HOOK,
-        )
+        ),
     )
 
+    for config_path in config_paths:
+        borgmatic.hooks.data_source.config.inject_pattern(
+            patterns,
+            borgmatic.borg.pattern.Pattern(
+                config_path,
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
+            ),
+        )
+
     return []
 
 
-def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, patterns, dry_run):
     '''
-    Given a bootstrap configuration dict, a configuration dict, the borgmatic runtime directory, and
-    whether this is a dry run, then remove the manifest file created above. If this is a dry run,
-    then don't actually remove anything.
+    Given a bootstrap configuration dict, a configuration dict, the borgmatic runtime directory, the
+    configured patterns, and whether this is a dry run, then remove the manifest file created above.
+    If this is a dry run, then don't actually remove anything.
     '''
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
@@ -86,7 +94,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         'bootstrap',
     )
     logger.debug(
-        f'Looking for bootstrap manifest files to remove in {manifest_glob}{dry_run_label}'
+        f'Looking for bootstrap manifest files to remove in {manifest_glob}{dry_run_label}',
     )
 
     for manifest_directory in glob.glob(manifest_glob):
@@ -96,19 +104,18 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         if dry_run:
             continue
 
-        try:
+        with contextlib.suppress(FileNotFoundError):
             os.remove(manifest_file_path)
-        except FileNotFoundError:
-            pass
 
-        try:
+        with contextlib.suppress(FileNotFoundError):
             os.rmdir(manifest_directory)
-        except FileNotFoundError:
-            pass
 
 
 def make_data_source_dump_patterns(
-    hook_config, config, borgmatic_runtime_directory, name=None
+    hook_config,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Restores are implemented via the separate, purpose-specific "bootstrap" action rather than the

borgmatic/hooks/data_source/btrfs.py

@@ -1,14 +1,16 @@
 import collections
+import functools
 import glob
-import json
 import logging
 import os
+import pathlib
 import shutil
 import subprocess
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.execute
+import borgmatic.hooks.data_source.config
 import borgmatic.hooks.data_source.snapshot
 
 logger = logging.getLogger(__name__)
@@ -21,38 +23,39 @@ def use_streaming(hook_config, config):  # pragma: no cover
     return False
 
 
-def get_subvolume_mount_points(findmnt_command):
+BTRFS_SUBVOLUME_INODE_NUMBER = 256
+
+
+@functools.cache
+def path_is_a_subvolume(path):
     '''
-    Given a findmnt command to run, get all sorted Btrfs subvolume mount points.
+    Given a path, return whether it is a Btrfs subvolume. Return False if the path doesn't exist.
+
+    As a performance optimization, multiple calls to this function with the same path are cached.
     '''
-    findmnt_output = borgmatic.execute.execute_command_and_capture_output(
-        tuple(findmnt_command.split(' '))
-        + (
-            '-t',  # Filesystem type.
-            'btrfs',
-            '--json',
-            '--list',  # Request a flat list instead of a nested subvolume hierarchy.
-        ),
-        close_fds=True,
-    )
+    if path == os.environ.get('BTRFS_TEST_SUBVOLUME_PATH'):  # pragma: no cover
+        return True
 
     try:
-        return tuple(
-            sorted(filesystem['target'] for filesystem in json.loads(findmnt_output)['filesystems'])
-        )
-    except json.JSONDecodeError as error:
-        raise ValueError(f'Invalid {findmnt_command} JSON output: {error}')
-    except KeyError as error:
-        raise ValueError(f'Invalid {findmnt_command} output: Missing key "{error}"')
-
-
-Subvolume = collections.namedtuple('Subvolume', ('path', 'contained_patterns'), defaults=((),))
+        return os.stat(path).st_ino == BTRFS_SUBVOLUME_INODE_NUMBER
+    except FileNotFoundError:
+        return False
 
 
+@functools.cache
 def get_subvolume_property(btrfs_command, subvolume_path, property_name):
+    '''
+    Given a btrfs command, a subvolume path, and a property name to lookup, return the value of the
+    corresponding property.
+
+    Raise subprocess.CalledProcessError if the btrfs command errors.
+
+    As a performance optimization, multiple calls to this function with the same arguments are
+    cached.
+    '''
     output = borgmatic.execute.execute_command_and_capture_output(
-        tuple(btrfs_command.split(' '))
-        + (
+        (
+            *btrfs_command.split(' '),
             'property',
             'get',
             '-t',  # Type.
@@ -74,56 +77,91 @@ def get_subvolume_property(btrfs_command, subvolume_path, property_name):
     }.get(value, value)
 
 
-def omit_read_only_subvolume_mount_points(btrfs_command, subvolume_paths):
+def get_containing_subvolume_path(btrfs_command, path):
     '''
-    Given a Btrfs command to run and a sequence of Btrfs subvolume mount points, filter them down to
-    just those that are read-write. The idea is that Btrfs can't actually snapshot a read-only
-    subvolume, so we should just ignore them.
+    Given a btrfs command and a path, return the subvolume path that contains the given path (or is
+    the same as the path).
+
+    If there is no such subvolume path or the containing subvolume is read-only, return None.
     '''
-    retained_subvolume_paths = []
+    # Probe the given pattern's path and all of its parents, grandparents, etc. to try to find a
+    # Btrfs subvolume.
+    for candidate_path in (
+        path,
+        *tuple(str(ancestor) for ancestor in pathlib.PurePath(path).parents),
+    ):
+        if not path_is_a_subvolume(candidate_path):
+            continue
 
-    for subvolume_path in subvolume_paths:
-        if get_subvolume_property(btrfs_command, subvolume_path, 'ro'):
-            logger.debug(f'Ignoring Btrfs subvolume {subvolume_path} because it is read-only')
-        else:
-            retained_subvolume_paths.append(subvolume_path)
+        try:
+            if get_subvolume_property(btrfs_command, candidate_path, 'ro'):
+                logger.debug(f'Ignoring Btrfs subvolume {candidate_path} because it is read-only')
 
-    return tuple(retained_subvolume_paths)
+                return None
+
+            logger.debug(f'Path {candidate_path} is a Btrfs subvolume')
+
+            return candidate_path
+        except subprocess.CalledProcessError as error:
+            logger.debug(
+                f'Error determining read-only status of Btrfs subvolume {candidate_path}: {error}',
+            )
+
+            return None
+
+    return None
 
 
-def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
+def get_all_subvolume_paths(btrfs_command, patterns):
+    '''
+    Given a btrfs command and a sequence of patterns, get the sorted paths for all Btrfs subvolumes
+    containing those patterns.
+    '''
+    return tuple(
+        sorted(
+            {
+                subvolume_path
+                for pattern in patterns
+                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                if pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                for subvolume_path in (get_containing_subvolume_path(btrfs_command, pattern.path),)
+                if subvolume_path
+            }
+        ),
+    )
+
+
+Subvolume = collections.namedtuple('Subvolume', ('path', 'contained_patterns'), defaults=((),))
+
+
+def get_subvolumes(btrfs_command, patterns):
     '''
     Given a Btrfs command to run and a sequence of configured patterns, find the intersection
-    between the current Btrfs filesystem and subvolume mount points and the paths of any patterns.
-    The idea is that these pattern paths represent the requested subvolumes to snapshot.
+    between the current Btrfs filesystem/subvolume paths and the paths of any patterns. The idea is
+    that these pattern paths represent the requested subvolumes to snapshot.
 
     Only include subvolumes that contain at least one root pattern sourced from borgmatic
-    configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then
-    return all subvolumes instead, sorted by path.
+    configuration (as opposed to generated elsewhere in borgmatic).
 
-    Return the result as a sequence of matching subvolume mount points.
+    Return the result as a sequence of matching Subvolume instances.
     '''
     candidate_patterns = set(patterns or ())
     subvolumes = []
 
-    # For each subvolume mount point, match it against the given patterns to find the subvolumes to
-    # backup. Sort the subvolumes from longest to shortest mount points, so longer mount points get
+    # For each subvolume path, match it against the given patterns to find the subvolumes to
+    # backup. Sort the subvolumes from longest to shortest mount points, so longer subvolumes get
     # a whack at the candidate pattern piñata before their parents do. (Patterns are consumed during
     # this process, so no two subvolumes end up with the same contained patterns.)
-    for mount_point in reversed(
-        omit_read_only_subvolume_mount_points(
-            btrfs_command, get_subvolume_mount_points(findmnt_command)
-        )
-    ):
+    for subvolume_path in reversed(get_all_subvolume_paths(btrfs_command, patterns)):
         subvolumes.extend(
-            Subvolume(mount_point, contained_patterns)
+            Subvolume(subvolume_path, contained_patterns)
             for contained_patterns in (
                 borgmatic.hooks.data_source.snapshot.get_contained_patterns(
-                    mount_point, candidate_patterns
+                    subvolume_path,
+                    candidate_patterns,
                 ),
             )
-            if patterns is None
-            or any(
+            if any(
                 pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
                 and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
                 for pattern in contained_patterns
@@ -133,7 +171,7 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
     return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path))
 
 
-BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot-'
+BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot'
 
 
 def make_snapshot_path(subvolume_path):
@@ -142,7 +180,7 @@ def make_snapshot_path(subvolume_path):
     '''
     return os.path.join(
         subvolume_path,
-        f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
+        f'{BORGMATIC_SNAPSHOT_PREFIX}',
         # Included so that the snapshot ends up in the Borg archive at the "original" subvolume path.
     ) + subvolume_path.rstrip(os.path.sep)
 
@@ -155,16 +193,16 @@ def make_snapshot_exclude_pattern(subvolume_path):  # pragma: no cover
     directory within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and
     make a snapshot of it at:
 
-        /mnt/.borgmatic-snapshot-1234/mnt
+        /mnt/.borgmatic-snapshot/mnt
 
     ... then the snapshot itself will have an empty directory at:
 
-        /mnt/.borgmatic-snapshot-1234/mnt/.borgmatic-snapshot-1234
+        /mnt/.borgmatic-snapshot/mnt/.borgmatic-snapshot
 
     So to prevent that from ending up in the Borg archive, this function produces an exclude pattern
     to exclude that path.
     '''
-    snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
+    snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}'
 
     return borgmatic.borg.pattern.Pattern(
         os.path.join(
@@ -197,8 +235,11 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
 
     rewritten_path = initial_caret + os.path.join(
         subvolume_path,
-        f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
-        '.',  # Borg 1.4+ "slashdot" hack.
+        f'{BORGMATIC_SNAPSHOT_PREFIX}',
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )
@@ -220,11 +261,10 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma:
     os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        tuple(btrfs_command.split(' '))
-        + (
+        (
+            *btrfs_command.split(' '),
             'subvolume',
             'snapshot',
-            '-r',  # Read-only.
             subvolume_path,
             snapshot_path,
         ),
@@ -244,9 +284,9 @@ def dump_data_sources(
     '''
     Given a Btrfs configuration dict, a configuration dict, the borgmatic configuration file paths,
     the borgmatic runtime directory, the configured patterns, and whether this is a dry run,
-    auto-detect and snapshot any Btrfs subvolume mount points listed in the given patterns. Also
-    update those patterns, replacing subvolume mount points with corresponding snapshot directories
-    so they get stored in the Borg archive instead.
+    auto-detect and snapshot any Btrfs subvolume paths listed in the given patterns. Also update
+    those patterns, replacing subvolume paths with corresponding snapshot directories so they get
+    stored in the Borg archive instead.
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.
 
@@ -255,11 +295,15 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}')
 
+    if 'findmnt_command' in hook_config:
+        logger.warning(
+            'The Btrfs "findmnt_command" option is deprecated and will be removed from a future release; findmnt is no longer used',
+        )
+
     # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those
     # patterns that came from actual user configuration (as opposed to, say, other hooks).
     btrfs_command = hook_config.get('btrfs_command', 'btrfs')
-    findmnt_command = hook_config.get('findmnt_command', 'findmnt')
-    subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns)
+    subvolumes = get_subvolumes(btrfs_command, patterns)
 
     if not subvolumes:
         logger.warning(f'No Btrfs subvolumes found to snapshot{dry_run_label}')
@@ -277,14 +321,11 @@ def dump_data_sources(
 
         for pattern in subvolume.contained_patterns:
             snapshot_pattern = make_borg_snapshot_pattern(subvolume.path, pattern)
+            borgmatic.hooks.data_source.config.replace_pattern(patterns, pattern, snapshot_pattern)
 
-            # Attempt to update the pattern in place, since pattern order matters to Borg.
-            try:
-                patterns[patterns.index(pattern)] = snapshot_pattern
-            except ValueError:
-                patterns.append(snapshot_pattern)
-
-        patterns.append(make_snapshot_exclude_pattern(subvolume.path))
+        borgmatic.hooks.data_source.config.inject_pattern(
+            patterns, make_snapshot_exclude_pattern(subvolume.path)
+        )
 
     return []
 
@@ -294,8 +335,8 @@ def delete_snapshot(btrfs_command, snapshot_path):  # pragma: no cover
     Given a Btrfs command to run and the name of a snapshot path, delete it.
     '''
     borgmatic.execute.execute_command(
-        tuple(btrfs_command.split(' '))
-        + (
+        (
+            *btrfs_command.split(' '),
             'subvolume',
             'delete',
             snapshot_path,
@@ -305,11 +346,12 @@ def delete_snapshot(btrfs_command, snapshot_path):  # pragma: no cover
     )
 
 
-def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, patterns, dry_run):
     '''
-    Given a Btrfs configuration dict, a configuration dict, the borgmatic runtime directory, and
-    whether this is a dry run, delete any Btrfs snapshots created by borgmatic. If this is a dry run
-    or Btrfs isn't configured in borgmatic's configuration, then don't actually remove anything.
+    Given a Btrfs configuration dict, a configuration dict, the borgmatic runtime directory, the
+    configured patterns, and whether this is a dry run, delete any Btrfs snapshots created by
+    borgmatic. If this is a dry run or Btrfs isn't configured in borgmatic's configuration, then
+    don't actually remove anything.
     '''
     if hook_config is None:
         return
@@ -317,10 +359,9 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     btrfs_command = hook_config.get('btrfs_command', 'btrfs')
-    findmnt_command = hook_config.get('findmnt_command', 'findmnt')
 
     try:
-        all_subvolumes = get_subvolumes(btrfs_command, findmnt_command)
+        all_subvolumes = get_subvolumes(btrfs_command, patterns)
     except FileNotFoundError as error:
         logger.debug(f'Could not find "{error.filename}" command')
         return
@@ -328,8 +369,8 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         logger.debug(error)
         return
 
-    # Reversing the sorted subvolumes ensures that we remove longer mount point paths of child
-    # subvolumes before the shorter mount point paths of parent subvolumes.
+    # Reversing the sorted subvolumes ensures that we remove longer paths of child subvolumes before
+    # the shorter paths of parent subvolumes.
     for subvolume in reversed(all_subvolumes):
         subvolume_snapshots_glob = borgmatic.config.paths.replace_temporary_subdirectory_with_glob(
             os.path.normpath(make_snapshot_path(subvolume.path)),
@@ -337,7 +378,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         )
 
         logger.debug(
-            f'Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}'
+            f'Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}',
         )
 
         for snapshot_path in glob.glob(subvolume_snapshots_glob):
@@ -367,7 +408,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
 
 
 def make_data_source_dump_patterns(
-    hook_config, config, borgmatic_runtime_directory, name=None
+    hook_config,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Restores aren't implemented, because stored files can be extracted directly with "extract".

borgmatic/hooks/data_source/config.py

@@ -0,0 +1,153 @@
+import json
+import logging
+import shutil
+import subprocess
+
+import borgmatic.borg.pattern
+from borgmatic.execute import execute_command_and_capture_output
+
+IS_A_HOOK = False
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_database_option(option, data_source, connection_params=None, restore=False):
+    '''
+    Resolves a database option from the given data source configuration dict and
+    connection parameters dict. If restore is set to True it will consider the
+    `restore_<option>` instead.
+
+    Returns the resolved option or None. Can raise a ValueError if the hostname lookup
+    results in a container IP check.
+    '''
+    # Special case `hostname` since it overlaps with `container`
+    if option == 'hostname':
+        return get_hostname_from_config(data_source, connection_params, restore)
+    if connection_params and (value := connection_params.get(option)):
+        return value
+    if restore and f'restore_{option}' in data_source:
+        return data_source[f'restore_{option}']
+
+    return data_source.get(option)
+
+
+def get_hostname_from_config(data_source, connection_params=None, restore=False):
+    '''
+    Specialisation of `resolve_database_option` to handle the extra complexity of
+    the hostname option to also handle containers.
+
+    Returns a hostname/IP or raises an ValueError if a container IP lookup fails.
+    '''
+    # connection params win, full stop
+    if connection_params:
+        if container := connection_params.get('container'):
+            return get_ip_from_container(container)
+        if hostname := connection_params.get('hostname'):
+            return hostname
+    # ... then try the restore config
+    if restore:
+        if 'restore_container' in data_source:
+            return get_ip_from_container(data_source['restore_container'])
+        if 'restore_hostname' in data_source:
+            return data_source['restore_hostname']
+    # ... and finally fall back to the normal options
+    if 'container' in data_source:
+        return get_ip_from_container(data_source['container'])
+
+    return data_source.get('hostname')
+
+
+def get_ip_from_container(container):
+    '''
+    Determine the IP for a given container name via podman and docker.
+
+    Returns an IP or raises a ValueError if the lookup fails.
+    '''
+    engines = (shutil.which(engine) for engine in ('docker', 'podman'))
+    engines = [engine for engine in engines if engine]
+
+    if not engines:
+        raise ValueError("Neither 'docker' nor 'podman' could be found on the system")
+
+    last_error = None
+    for engine in engines:
+        try:
+            output = execute_command_and_capture_output(
+                (
+                    engine,
+                    'container',
+                    'inspect',
+                    '--format={{json .NetworkSettings}}',
+                    container,
+                )
+            )
+        except subprocess.CalledProcessError as error:
+            last_error = error
+            logger.debug(f"Could not find container '{container}' with engine '{engine}'")
+            continue  # Container does not exist
+
+        try:
+            network_data = json.loads(output.strip())
+        except json.JSONDecodeError as e:
+            raise ValueError(f'Could not decode JSON output from {engine}') from e
+        if main_ip := network_data.get('IPAddress'):
+            return main_ip
+        # No main IP found, try the networks
+        for network in network_data.get('Networks', {}).values():
+            if ip := network.get('IPAddress'):
+                return ip
+
+    if last_error:
+        raise last_error
+
+    raise ValueError(
+        f"Could not determine ip address for container '{container}'; running in host mode or userspace networking?"
+    )
+
+
+def inject_pattern(patterns, data_source_pattern, override_excludes=True):
+    '''
+    Given a list of borgmatic.borg.pattern.Pattern instances representing the configured patterns,
+    insert the given data source pattern at the start of the list. The idea is that borgmatic is
+    injecting its own custom pattern specific to a data source hook into the user's configured
+    patterns so that the hook's data gets included in the backup.
+
+    As part of this injection, if the data source pattern is a root pattern and override_excludes is
+    True, also insert an "include" version of the given root pattern, in an attempt to preempt any
+    of the user's configured exclude patterns that may follow. The is to support use cases like
+    borgmatic injecting its own patterns for things like database dumps or bootstrap metadata, where
+    we don't want them to get accidentally excluded.
+    '''
+    if data_source_pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT and override_excludes:
+        patterns.insert(
+            0,
+            borgmatic.borg.pattern.Pattern(
+                path=data_source_pattern.path,
+                type=borgmatic.borg.pattern.Pattern_type.INCLUDE,
+                style=data_source_pattern.style,
+                device=data_source_pattern.device,
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
+            ),
+        )
+
+    patterns.insert(0, data_source_pattern)
+
+
+def replace_pattern(patterns, pattern_to_replace, data_source_pattern):
+    '''
+    Given a list of borgmatic.borg.pattern.Pattern instances representing the configured patterns,
+    replace the given pattern with the given data source pattern. The idea is that borgmatic is
+    replacing a configured pattern with its own modified pattern specific to a data source hook so
+    that the hook's data gets included in the backup.
+
+    If the pattern to replace can't be found in the given patterns, then just inject the data source
+    pattern at the start of the list.
+    '''
+    try:
+        index = patterns.index(pattern_to_replace)
+    except ValueError:
+        inject_pattern(patterns, data_source_pattern, override_excludes=False)
+
+        return
+
+    patterns[index] = data_source_pattern

borgmatic/hooks/data_source/dump.py

@@ -1,8 +1,11 @@
 import fnmatch
+import json
 import logging
 import os
 import shutil
 
+import borgmatic.actions.restore
+
 logger = logging.getLogger(__name__)
 
 IS_A_HOOK = False
@@ -16,7 +19,9 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam
     return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
 
 
-def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
+def make_data_source_dump_filename(
+    dump_path, name, hostname=None, port=None, container=None, label=None
+):
     '''
     Based on the given dump directory path, data source name, hostname, and port, return a filename
     to use for the data source dump. The hostname defaults to localhost.
@@ -26,9 +31,56 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
     if os.path.sep in name:
         raise ValueError(f'Invalid data source name {name}')
 
-    return os.path.join(
-        dump_path, (hostname or 'localhost') + ('' if port is None else f':{port}'), name
-    )
+    if container:
+        hostname = container
+    elif not hostname:
+        hostname = 'localhost'
+
+    identifier = label if label else hostname + ('' if port is None else f':{port}')
+
+    return os.path.join(dump_path, identifier, name)
+
+
+def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata):
+    '''
+    Given the borgmatic runtime directory, a data source hook name, and a sequence of
+    borgmatic.actions.restore.Dump instances of dump metadata, write a metadata file describing all
+    of those dumps. This metadata is being dumped so that it's available upon restore, e.g. to
+    support the user selecting which data source(s) should be restored.
+
+    Raise ValueError if writing to the file results in an operating system error.
+    '''
+    dumps_metadata_path = os.path.join(borgmatic_runtime_directory, hook_name, 'dumps.json')
+
+    try:
+        with open(dumps_metadata_path, 'w', encoding='utf-8') as metadata_file:
+            json.dump(
+                {
+                    'dumps': [dump._asdict() for dump in dumps_metadata],
+                },
+                metadata_file,
+                sort_keys=True,
+            )
+    except OSError as error:
+        raise ValueError(f'Error writing to dumps metadata at {dumps_metadata_path}: {error}')
+
+
+def parse_data_source_dumps_metadata(dumps_json, dumps_metadata_path):
+    '''
+    Given a dumps metadata JSON string as extracted from an archive and its path within the archive,
+    parse it into a tuple of borgmatic.actions.restore.Dump instances and return them.
+
+    Raise ValueError if parsing the JSON results in a JSON decode error or the data does not have
+    the expected keys.
+    '''
+    try:
+        return tuple(
+            borgmatic.actions.restore.Dump(**dump) for dump in json.loads(dumps_json)['dumps']
+        )
+    except (json.JSONDecodeError, TypeError, KeyError) as error:
+        raise ValueError(
+            f'Cannot read archive data source dumps metadata at {dumps_metadata_path} due to invalid JSON: {error}',
+        )
 
 
 def create_parent_directory_for_dump(dump_path):
@@ -67,10 +119,11 @@ def convert_glob_patterns_to_borg_pattern(patterns):
     Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg
     regular expression archive pattern as a single string like "re:etc/.*|tmp/.*".
     '''
-    # Remove the "\Z" generated by fnmatch.translate() because we don't want the pattern to match
-    # only at the end of a path, as directory format dumps require extracting files with paths
+    # Remove the "\z" or "\Z" generated by fnmatch.translate() because we don't want the pattern to
+    # match only at the end of a path, as directory format dumps require extracting files with paths
     # longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match
     # paths like "borgmatic/*/foo_databases/test/toc.dat"
     return 're:' + '|'.join(
-        fnmatch.translate(pattern.lstrip('/')).replace('\\Z', '') for pattern in patterns
+        fnmatch.translate(pattern.lstrip('/')).replace('\\z', '').replace('\\Z', '')
+        for pattern in patterns
     )

borgmatic/hooks/data_source/lvm.py

@@ -10,6 +10,7 @@ import subprocess
 import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.execute
+import borgmatic.hooks.data_source.config
 import borgmatic.hooks.data_source.snapshot
 
 logger = logging.getLogger(__name__)
@@ -24,7 +25,8 @@ def use_streaming(hook_config, config):  # pragma: no cover
 
 BORGMATIC_SNAPSHOT_PREFIX = 'borgmatic-'
 Logical_volume = collections.namedtuple(
-    'Logical_volume', ('name', 'device_path', 'mount_point', 'contained_patterns')
+    'Logical_volume',
+    ('name', 'device_path', 'mount_point', 'contained_patterns'),
 )
 
 
@@ -44,15 +46,15 @@ def get_logical_volumes(lsblk_command, patterns=None):
         devices_info = json.loads(
             borgmatic.execute.execute_command_and_capture_output(
                 # Use lsblk instead of lvs here because lvs can't show active mounts.
-                tuple(lsblk_command.split(' '))
-                + (
+                (
+                    *lsblk_command.split(' '),
                     '--output',
                     'name,path,mountpoint,type',
                     '--json',
                     '--list',
                 ),
                 close_fds=True,
-            )
+            ),
         )
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {lsblk_command} JSON output: {error}')
@@ -73,7 +75,8 @@ def get_logical_volumes(lsblk_command, patterns=None):
             if device['mountpoint'] and device['type'] == 'lvm'
             for contained_patterns in (
                 borgmatic.hooks.data_source.snapshot.get_contained_patterns(
-                    device['mountpoint'], candidate_patterns
+                    device['mountpoint'],
+                    candidate_patterns,
                 ),
             )
             if not patterns
@@ -98,13 +101,13 @@ def snapshot_logical_volume(
     snapshot, and a snapshot size string, create a new LVM snapshot.
     '''
     borgmatic.execute.execute_command(
-        tuple(lvcreate_command.split(' '))
-        + (
+        (
+            *lvcreate_command.split(' '),
             '--snapshot',
             ('--extents' if '%' in snapshot_size else '--size'),
             snapshot_size,
             '--permission',
-            'r',  # Read-only.
+            'rw',  # Read-write in case an ext4 filesystem has orphaned files that need recovery.
             '--name',
             snapshot_name,
             logical_volume_device,
@@ -123,8 +126,8 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # prag
     os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        tuple(mount_command.split(' '))
-        + (
+        (
+            *mount_command.split(' '),
             '-o',
             'ro',
             snapshot_device,
@@ -162,9 +165,12 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc
         # /var/spool would result in overlapping snapshot patterns and therefore colliding mount
         # attempts.
         hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
-            MOUNT_POINT_HASH_LENGTH
+            MOUNT_POINT_HASH_LENGTH,
         ),
-        '.',  # Borg 1.4+ "slashdot" hack.
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )
@@ -218,7 +224,7 @@ def dump_data_sources(
     for logical_volume in requested_logical_volumes:
         snapshot_name = f'{logical_volume.name}_{snapshot_suffix}'
         logger.debug(
-            f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}'
+            f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}',
         )
 
         if not dry_run:
@@ -233,7 +239,8 @@ def dump_data_sources(
         if not dry_run:
             try:
                 snapshot = get_snapshots(
-                    hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name
+                    hook_config.get('lvs_command', 'lvs'),
+                    snapshot_name=snapshot_name,
                 )[0]
             except IndexError:
                 raise ValueError(f'Cannot find LVM snapshot {snapshot_name}')
@@ -244,32 +251,32 @@ def dump_data_sources(
             normalized_runtime_directory,
             'lvm_snapshots',
             hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
-                MOUNT_POINT_HASH_LENGTH
+                MOUNT_POINT_HASH_LENGTH,
             ),
             logical_volume.mount_point.lstrip(os.path.sep),
         )
 
         logger.debug(
-            f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}'
+            f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}',
         )
 
         if dry_run:
             continue
 
         mount_snapshot(
-            hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path
+            hook_config.get('mount_command', 'mount'),
+            snapshot.device_path,
+            snapshot_mount_path,
         )
 
         for pattern in logical_volume.contained_patterns:
             snapshot_pattern = make_borg_snapshot_pattern(
-                pattern, logical_volume, normalized_runtime_directory
+                pattern,
+                logical_volume,
+                normalized_runtime_directory,
             )
 
-            # Attempt to update the pattern in place, since pattern order matters to Borg.
-            try:
-                patterns[patterns.index(pattern)] = snapshot_pattern
-            except ValueError:
-                patterns.append(snapshot_pattern)
+            borgmatic.hooks.data_source.config.replace_pattern(patterns, pattern, snapshot_pattern)
 
     return []
 
@@ -279,7 +286,7 @@ def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
     Given a umount command to run and the mount path of a snapshot, unmount it.
     '''
     borgmatic.execute.execute_command(
-        tuple(umount_command.split(' ')) + (snapshot_mount_path,),
+        (*umount_command.split(' '), snapshot_mount_path),
         output_log_level=logging.DEBUG,
         close_fds=True,
     )
@@ -290,8 +297,8 @@ def remove_snapshot(lvremove_command, snapshot_device_path):  # pragma: no cover
     Given an lvremove command to run and the device path of a snapshot, remove it it.
     '''
     borgmatic.execute.execute_command(
-        tuple(lvremove_command.split(' '))
-        + (
+        (
+            *lvremove_command.split(' '),
             '--force',  # Suppress an interactive "are you sure?" type prompt.
             snapshot_device_path,
         ),
@@ -316,8 +323,8 @@ def get_snapshots(lvs_command, snapshot_name=None):
         snapshot_info = json.loads(
             borgmatic.execute.execute_command_and_capture_output(
                 # Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
-                tuple(lvs_command.split(' '))
-                + (
+                (
+                    *lvs_command.split(' '),
                     '--report-format',
                     'json',
                     '--options',
@@ -326,7 +333,7 @@ def get_snapshots(lvs_command, snapshot_name=None):
                     'lv_attr =~ ^s',  # Filter to just snapshots.
                 ),
                 close_fds=True,
-            )
+            ),
         )
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {lvs_command} JSON output: {error}')
@@ -343,12 +350,12 @@ def get_snapshots(lvs_command, snapshot_name=None):
         raise ValueError(f'Invalid {lvs_command} output: Missing key "{error}"')
 
 
-def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, patterns, dry_run):  # noqa: PLR0912
     '''
-    Given an LVM configuration dict, a configuration dict, the borgmatic runtime directory, and
-    whether this is a dry run, unmount and delete any LVM snapshots created by borgmatic. If this is
-    a dry run or LVM isn't configured in borgmatic's configuration, then don't actually remove
-    anything.
+    Given an LVM configuration dict, a configuration dict, the borgmatic runtime directory, the
+    configured patterns, and whether this is a dry run, unmount and delete any LVM snapshots created
+    by borgmatic. If this is a dry run or LVM isn't configured in borgmatic's configuration, then
+    don't actually remove anything.
     '''
     if hook_config is None:
         return
@@ -381,7 +388,8 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
 
         for logical_volume in logical_volumes:
             snapshot_mount_path = os.path.join(
-                snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep)
+                snapshots_directory,
+                logical_volume.mount_point.lstrip(os.path.sep),
             )
 
             # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
@@ -440,7 +448,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
 
 
 def make_data_source_dump_patterns(
-    hook_config, config, borgmatic_runtime_directory, name=None
+    hook_config,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Restores aren't implemented, because stored files can be extracted directly with "extract".

borgmatic/hooks/data_source/mariadb.py

@@ -7,11 +7,13 @@ import shlex
 import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
+import borgmatic.hooks.data_source.config
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -24,7 +26,7 @@ def make_dump_path(base_directory):  # pragma: no cover
     return dump.make_data_source_dump_path(base_directory, 'mariadb_databases')
 
 
-DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile('^--defaults-extra-file=(?P<filename>.*)$')
+DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile(r'^--defaults-extra-file=(?P<filename>.*)$')
 
 
 def parse_extra_options(extra_options):
@@ -71,7 +73,7 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file
         (
             (f'user={username}' if username is not None else ''),
             (f'password="{escaped_password}"' if escaped_password is not None else ''),
-        )
+        ),
     ).strip()
 
     if not values:
@@ -94,7 +96,7 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file
     include = f'!include {defaults_extra_filename}\n' if defaults_extra_filename else ''
 
     read_file_descriptor, write_file_descriptor = os.pipe()
-    os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode('utf-8'))
+    os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode())
     os.close(write_file_descriptor)
 
     # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database
@@ -111,8 +113,16 @@ def database_names_to_dump(database, config, username, password, environment, dr
     names to dump. In the case of "all", query for the names of databases on the configured host and
     return them, excluding any system databases that will cause problems during restore.
     '''
+    skip_names = database.get('skip_names')
+
     if database['name'] != 'all':
+        if skip_names:
+            logger.warning(
+                f'For MariaDB database {database["name"]}, ignoring the "skip_names" option, which is only supported for database "all"'
+            )
+
         return (database['name'],)
+
     if dry_run:
         return ()
 
@@ -121,6 +131,9 @@ def database_names_to_dump(database, config, username, password, environment, dr
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options'))
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
+    socket_path = database.get('socket_path')
+
     show_command = (
         mariadb_show_command
         + (
@@ -129,9 +142,10 @@ def database_names_to_dump(database, config, username, password, environment, dr
             else ()
         )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--socket', socket_path) if socket_path else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -141,12 +155,20 @@ def database_names_to_dump(database, config, username, password, environment, dr
 
     logger.debug('Querying for "all" MariaDB databases to dump')
 
-    show_output = execute_command_and_capture_output(show_command, environment=environment)
+    if skip_names:
+        logger.debug(f'Skipping database names: {", ".join(skip_names)}')
+
+    show_output = execute_command_and_capture_output(
+        show_command,
+        environment=environment,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+    )
 
     return tuple(
         show_name
         for show_name in show_output.strip().splitlines()
         if show_name not in SYSTEM_DATABASE_NAMES
+        if not skip_names or show_name not in skip_names
     )
 
 
@@ -175,13 +197,15 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
-        database.get('port'),
+        hostname=database.get('hostname'),
+        port=database.get('port'),
+        container=database.get('container'),
+        label=database.get('label'),
     )
 
     if os.path.exists(dump_filename):
         logger.warning(
-            f'Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
+            f'Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}',
         )
         return None
 
@@ -191,6 +215,9 @@ def execute_dump_command(
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('options'))
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
+    socket_path = database.get('socket_path')
+
     dump_command = (
         mariadb_dump_command
         + (
@@ -200,9 +227,11 @@ def execute_dump_command(
         )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + ('--single-transaction',)
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--socket', socket_path) if socket_path else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -221,6 +250,7 @@ def execute_dump_command(
         dump_command,
         environment=environment,
         run_to_completion=False,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
     )
 
 
@@ -256,16 +286,19 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping MariaDB databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         username = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('username'), config
+            database.get('username'),
+            config,
         )
         password = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('password'), config
+            database.get('password'),
+            config,
         )
         environment = dict(
             os.environ,
@@ -276,7 +309,12 @@ def dump_data_sources(
             ),
         )
         dump_database_names = database_names_to_dump(
-            database, config, username, password, environment, dry_run
+            database,
+            config,
+            username,
+            password,
+            environment,
+            dry_run,
         )
 
         if not dump_database_names:
@@ -286,9 +324,19 @@ def dump_data_sources(
             raise ValueError('Cannot find any MariaDB databases to dump.')
 
         if database['name'] == 'all' and database.get('format'):
-            for dump_name in dump_database_names:
+            for database_name in dump_database_names:
+                dumps_metadata.append(
+                    borgmatic.actions.restore.Dump(
+                        'mariadb_databases',
+                        database_name,
+                        database.get('hostname'),
+                        database.get('port'),
+                        database.get('label'),
+                        database.get('container'),
+                    )
+                )
                 renamed_database = copy.copy(database)
-                renamed_database['name'] = dump_name
+                renamed_database['name'] = database_name
                 processes.append(
                     execute_dump_command(
                         renamed_database,
@@ -296,13 +344,23 @@ def dump_data_sources(
                         username,
                         password,
                         dump_path,
-                        (dump_name,),
+                        (database_name,),
                         environment,
                         dry_run,
                         dry_run_label,
-                    )
+                    ),
                 )
         else:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'mariadb_databases',
+                    database['name'],
+                    database.get('hostname'),
+                    database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
+                )
+            )
             processes.append(
                 execute_dump_command(
                     database,
@@ -314,22 +372,30 @@ def dump_data_sources(
                     environment,
                     dry_run,
                     dry_run_label,
-                )
+                ),
             )
 
     if not dry_run:
-        patterns.append(
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mariadb_databases', dumps_metadata
+        )
+        borgmatic.hooks.data_source.config.inject_pattern(
+            patterns,
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mariadb_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return [process for process in processes if process]
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -340,7 +406,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -350,12 +419,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -376,24 +449,23 @@ def restore_data_source_dump(
     subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
-        'restore_hostname', data_source.get('hostname')
+    hostname = database_config.resolve_database_option(
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
+    port = database_config.resolve_database_option(
+        'port', data_source, connection_params, restore=True
     )
-    tls = data_source.get('restore_tls', data_source.get('tls'))
+    socket_path = database_config.resolve_database_option('socket_path', data_source, restore=True)
+    tls = database_config.resolve_database_option('tls', data_source, restore=True)
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or data_source.get('restore_username', data_source.get('username'))
+        database_config.resolve_database_option(
+            'username', data_source, connection_params, restore=True
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['password']
-            or data_source.get('restore_password', data_source.get('password'))
+        database_config.resolve_database_option(
+            'password', data_source, connection_params, restore=True
         ),
         config,
     )
@@ -403,6 +475,7 @@ def restore_data_source_dump(
     )
     extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options'))
     password_transport = data_source.get('password_transport', 'pipe')
+
     restore_command = (
         mariadb_restore_command
         + (
@@ -415,6 +488,7 @@ def restore_data_source_dump(
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--socket', socket_path) if socket_path else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if tls is True else ())
         + (('--skip-ssl',) if tls is False else ())
@@ -436,4 +510,5 @@ def restore_data_source_dump(
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         environment=environment,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
     )

borgmatic/hooks/data_source/mongodb.py

@@ -5,7 +5,9 @@ import shlex
 import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
+import borgmatic.hooks.data_source.config
 from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -53,14 +55,28 @@ def dump_data_sources(
     logger.info(f'Dumping MongoDB databases{dry_run_label}')
 
     processes = []
+    dumps_metadata = []
 
     for database in databases:
         name = database['name']
+        dumps_metadata.append(
+            borgmatic.actions.restore.Dump(
+                'mongodb_databases',
+                name,
+                database.get('hostname'),
+                database.get('port'),
+                database.get('label'),
+                database.get('container'),
+            )
+        )
+
         dump_filename = dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            database.get('hostname'),
-            database.get('port'),
+            hostname=database.get('hostname'),
+            port=database.get('port'),
+            container=database.get('container'),
+            label=database.get('label'),
         )
         dump_format = database.get('format', 'archive')
 
@@ -74,17 +90,32 @@ def dump_data_sources(
 
         if dump_format == 'directory':
             dump.create_parent_directory_for_dump(dump_filename)
-            execute_command(command, shell=True)
+            execute_command(  # noqa: S604
+                command,
+                shell=True,
+                working_directory=borgmatic.config.paths.get_working_directory(config),
+            )
         else:
             dump.create_named_pipe_for_dump(dump_filename)
-            processes.append(execute_command(command, shell=True, run_to_completion=False))
+            processes.append(
+                execute_command(  # noqa: S604
+                    command,
+                    shell=True,
+                    run_to_completion=False,
+                    working_directory=borgmatic.config.paths.get_working_directory(config),
+                ),
+            )
 
     if not dry_run:
-        patterns.append(
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mongodb_databases', dumps_metadata
+        )
+        borgmatic.hooks.data_source.config.inject_pattern(
+            patterns,
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mongodb_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return processes
@@ -102,7 +133,7 @@ def make_password_config_file(password):
     logger.debug('Writing MongoDB password to configuration file pipe')
 
     read_file_descriptor, write_file_descriptor = os.pipe()
-    os.write(write_file_descriptor, f'password: {password}'.encode('utf-8'))
+    os.write(write_file_descriptor, f'password: {password}'.encode())
     os.close(write_file_descriptor)
 
     # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database
@@ -123,18 +154,20 @@ def build_dump_command(database, config, dump_filename, dump_format):
     dump_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump')
     )
+    hostname = database_config.resolve_database_option('hostname', database)
     return (
         dump_command
         + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
-        + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+        + (('--host', shlex.quote(hostname)) if hostname else ())
         + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
         + (
             (
                 '--username',
                 shlex.quote(
                     borgmatic.hooks.credential.parse.resolve_credential(
-                        database['username'], config
-                    )
+                        database['username'],
+                        config,
+                    ),
                 ),
             )
             if 'username' in database
@@ -142,7 +175,10 @@ def build_dump_command(database, config, dump_filename, dump_format):
         )
         + (('--config', make_password_config_file(password)) if password else ())
         + (
-            ('--authenticationDatabase', shlex.quote(database['authentication_database']))
+            (
+                '--authenticationDatabase',
+                shlex.quote(database['authentication_database']),
+            )
             if 'authentication_database' in database
             else ()
         )
@@ -157,7 +193,11 @@ def build_dump_command(database, config, dump_filename, dump_format):
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -168,7 +208,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -178,12 +221,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -211,13 +258,20 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
+        hostname=data_source.get('hostname'),
+        port=data_source.get('port'),
+        container=data_source.get('container'),
+        label=data_source.get('label'),
     )
     restore_command = build_restore_command(
-        extract_process, data_source, config, dump_filename, connection_params
+        extract_process,
+        data_source,
+        config,
+        dump_filename,
+        connection_params,
     )
 
-    logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}")
+    logger.debug(f'Restoring MongoDB database {data_source["name"]}{dry_run_label}')
     if dry_run:
         return
 
@@ -228,6 +282,7 @@ def restore_data_source_dump(
         [extract_process] if extract_process else [],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout if extract_process else None,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
     )
 
 
@@ -235,47 +290,56 @@ def build_restore_command(extract_process, database, config, dump_filename, conn
     '''
     Return the custom mongorestore_command from a single database configuration.
     '''
-    hostname = connection_params['hostname'] or database.get(
-        'restore_hostname', database.get('hostname')
+    hostname = database_config.resolve_database_option(
+        'hostname', database, connection_params, restore=True
+    )
+    port = database_config.resolve_database_option(
+        'port', database, connection_params, restore=True
     )
-    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or database.get('restore_username', database.get('username'))
+        database_config.resolve_database_option(
+            'username', database, connection_params, restore=True
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['password']
-            or database.get('restore_password', database.get('password'))
+        database_config.resolve_database_option(
+            'password', database, connection_params, restore=True
         ),
         config,
     )
 
-    command = list(
+    command = [
         shlex.quote(part)
         for part in shlex.split(database.get('mongorestore_command') or 'mongorestore')
-    )
+    ]
+
     if extract_process:
         command.append('--archive')
     else:
         command.extend(('--dir', dump_filename))
+
     if database['name'] != 'all':
         command.extend(('--drop',))
+
     if hostname:
         command.extend(('--host', hostname))
+
     if port:
         command.extend(('--port', str(port)))
+
     if username:
         command.extend(('--username', username))
+
     if password:
         command.extend(('--config', make_password_config_file(password)))
+
     if 'authentication_database' in database:
         command.extend(('--authenticationDatabase', database['authentication_database']))
+
     if 'restore_options' in database:
         command.extend(database['restore_options'].split(' '))
+
     if database.get('schemas'):
         for schema in database['schemas']:
             command.extend(('--nsInclude', schema))

borgmatic/hooks/data_source/mysql.py

@@ -6,12 +6,14 @@ import shlex
 import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
+import borgmatic.hooks.data_source.config
 import borgmatic.hooks.data_source.mariadb
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -34,8 +36,16 @@ def database_names_to_dump(database, config, username, password, environment, dr
     names to dump. In the case of "all", query for the names of databases on the configured host and
     return them, excluding any system databases that will cause problems during restore.
     '''
+    skip_names = database.get('skip_names')
+
     if database['name'] != 'all':
+        if skip_names:
+            logger.warning(
+                f'For MySQL database {database["name"]}, ignoring the "skip_names" option, which is only supported for database "all"'
+            )
+
         return (database['name'],)
+
     if dry_run:
         return ()
 
@@ -46,19 +56,25 @@ def database_names_to_dump(database, config, username, password, environment, dr
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options'))
     )
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
+    socket_path = database.get('socket_path')
+
     show_command = (
         mysql_show_command
         + (
             borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
-                username, password, defaults_extra_filename
+                username,
+                password,
+                defaults_extra_filename,
             )
             if password_transport == 'pipe'
             else ()
         )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--socket', socket_path) if socket_path else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -68,12 +84,20 @@ def database_names_to_dump(database, config, username, password, environment, dr
 
     logger.debug('Querying for "all" MySQL databases to dump')
 
-    show_output = execute_command_and_capture_output(show_command, environment=environment)
+    if skip_names:
+        logger.debug(f'Skipping database names: {", ".join(skip_names)}')
+
+    show_output = execute_command_and_capture_output(
+        show_command,
+        environment=environment,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+    )
 
     return tuple(
         show_name
         for show_name in show_output.strip().splitlines()
         if show_name not in SYSTEM_DATABASE_NAMES
+        if not skip_names or show_name not in skip_names
     )
 
 
@@ -99,13 +123,15 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
-        database.get('port'),
+        hostname=database.get('hostname'),
+        port=database.get('port'),
+        container=database.get('container'),
+        label=database.get('label'),
     )
 
     if os.path.exists(dump_filename):
         logger.warning(
-            f'Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}'
+            f'Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}',
         )
         return None
 
@@ -116,20 +142,27 @@ def execute_dump_command(
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options'))
     )
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
+    socket_path = database.get('socket_path')
+
     dump_command = (
         mysql_dump_command
         + (
             borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
-                username, password, defaults_extra_filename
+                username,
+                password,
+                defaults_extra_filename,
             )
             if password_transport == 'pipe'
             else ()
         )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + ('--single-transaction',)
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--socket', socket_path) if socket_path else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -148,6 +181,7 @@ def execute_dump_command(
         dump_command,
         environment=environment,
         run_to_completion=False,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
     )
 
 
@@ -183,16 +217,19 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping MySQL databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         username = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('username'), config
+            database.get('username'),
+            config,
         )
         password = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('password'), config
+            database.get('password'),
+            config,
         )
         environment = dict(
             os.environ,
@@ -203,7 +240,12 @@ def dump_data_sources(
             ),
         )
         dump_database_names = database_names_to_dump(
-            database, config, username, password, environment, dry_run
+            database,
+            config,
+            username,
+            password,
+            environment,
+            dry_run,
         )
 
         if not dump_database_names:
@@ -213,9 +255,19 @@ def dump_data_sources(
             raise ValueError('Cannot find any MySQL databases to dump.')
 
         if database['name'] == 'all' and database.get('format'):
-            for dump_name in dump_database_names:
+            for database_name in dump_database_names:
+                dumps_metadata.append(
+                    borgmatic.actions.restore.Dump(
+                        'mysql_databases',
+                        database_name,
+                        database.get('hostname'),
+                        database.get('port'),
+                        database.get('label'),
+                        database.get('container'),
+                    )
+                )
                 renamed_database = copy.copy(database)
-                renamed_database['name'] = dump_name
+                renamed_database['name'] = database_name
                 processes.append(
                     execute_dump_command(
                         renamed_database,
@@ -223,13 +275,23 @@ def dump_data_sources(
                         username,
                         password,
                         dump_path,
-                        (dump_name,),
+                        (database_name,),
                         environment,
                         dry_run,
                         dry_run_label,
-                    )
+                    ),
                 )
         else:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'mysql_databases',
+                    database['name'],
+                    database.get('hostname'),
+                    database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
+                )
+            )
             processes.append(
                 execute_dump_command(
                     database,
@@ -241,22 +303,30 @@ def dump_data_sources(
                     environment,
                     dry_run,
                     dry_run_label,
-                )
+                ),
             )
 
     if not dry_run:
-        patterns.append(
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mysql_databases', dumps_metadata
+        )
+        borgmatic.hooks.data_source.config.inject_pattern(
+            patterns,
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mysql_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return [process for process in processes if process]
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -267,7 +337,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -277,12 +350,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -303,24 +380,23 @@ def restore_data_source_dump(
     subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
-        'restore_hostname', data_source.get('hostname')
+    hostname = database_config.resolve_database_option(
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
+    port = database_config.resolve_database_option(
+        'port', data_source, connection_params, restore=True
     )
-    tls = data_source.get('restore_tls', data_source.get('tls'))
+    socket_path = database_config.resolve_database_option('socket_path', data_source, restore=True)
+    tls = database_config.resolve_database_option('tls', data_source, restore=True)
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or data_source.get('restore_username', data_source.get('username'))
+        database_config.resolve_database_option(
+            'username', data_source, connection_params, restore=True
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['password']
-            or data_source.get('restore_password', data_source.get('password'))
+        database_config.resolve_database_option(
+            'password', data_source, connection_params, restore=True
         ),
         config,
     )
@@ -332,11 +408,14 @@ def restore_data_source_dump(
         borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options'))
     )
     password_transport = data_source.get('password_transport', 'pipe')
+
     restore_command = (
         mysql_restore_command
         + (
             borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
-                username, password, defaults_extra_filename
+                username,
+                password,
+                defaults_extra_filename,
             )
             if password_transport == 'pipe'
             else ()
@@ -346,6 +425,7 @@ def restore_data_source_dump(
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--socket', socket_path) if socket_path else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if tls is True else ())
         + (('--skip-ssl',) if tls is False else ())
@@ -367,4 +447,5 @@ def restore_data_source_dump(
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         environment=environment,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
     )

borgmatic/hooks/data_source/postgresql.py

@@ -8,11 +8,13 @@ import shlex
 import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
+import borgmatic.hooks.data_source.config
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -32,30 +34,28 @@ def make_environment(database, config, restore_connection_params=None):
     '''
     environment = dict(os.environ)
 
-    try:
-        if restore_connection_params:
-            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                (
-                    restore_connection_params.get('password')
-                    or database.get('restore_password', database['password'])
-                ),
-                config,
-            )
-        else:
-            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                database['password'], config
-            )
-    except (AttributeError, KeyError):
-        pass
+    password = database_config.resolve_database_option(
+        'password', database, restore_connection_params, restore=restore_connection_params
+    )
+
+    if password:
+        environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
+            password,
+            config,
+        )
 
     if 'ssl_mode' in database:
         environment['PGSSLMODE'] = database['ssl_mode']
+
     if 'ssl_cert' in database:
         environment['PGSSLCERT'] = database['ssl_cert']
+
     if 'ssl_key' in database:
         environment['PGSSLKEY'] = database['ssl_key']
+
     if 'ssl_root_cert' in database:
         environment['PGSSLROOTCERT'] = database['ssl_root_cert']
+
     if 'ssl_crl' in database:
         environment['PGSSLCRL'] = database['ssl_crl']
 
@@ -76,18 +76,21 @@ def database_names_to_dump(database, config, environment, dry_run):
 
     if requested_name != 'all':
         return (requested_name,)
+
     if not database.get('format'):
         return ('all',)
+
     if dry_run:
         return ()
 
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql')
     )
+    hostname = database_config.resolve_database_option('hostname', database)
     list_command = (
         psql_command
         + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (
             (
@@ -100,7 +103,11 @@ def database_names_to_dump(database, config, environment, dry_run):
         + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
     )
     logger.debug('Querying for "all" PostgreSQL databases to dump')
-    list_output = execute_command_and_capture_output(list_command, environment=environment)
+    list_output = execute_command_and_capture_output(
+        list_command,
+        environment=environment,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+    )
 
     return tuple(
         row[0]
@@ -143,12 +150,13 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping PostgreSQL databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
         environment = make_environment(database, config)
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
@@ -158,6 +166,16 @@ def dump_data_sources(
             raise ValueError('Cannot find any PostgreSQL databases to dump.')
 
         for database_name in dump_database_names:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'postgresql_databases',
+                    database_name,
+                    database.get('hostname'),
+                    database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
+                )
+            )
             dump_format = database.get('format', None if database_name == 'all' else 'custom')
             compression = database.get('compression')
             default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump'
@@ -168,15 +186,19 @@ def dump_data_sources(
             dump_filename = dump.make_data_source_dump_filename(
                 dump_path,
                 database_name,
-                database.get('hostname'),
-                database.get('port'),
+                hostname=database.get('hostname'),
+                port=database.get('port'),
+                container=database.get('container'),
+                label=database.get('label'),
             )
+
             if os.path.exists(dump_filename):
                 logger.warning(
-                    f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}'
+                    f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}',
                 )
                 continue
 
+            hostname = database_config.resolve_database_option('hostname', database)
             command = (
                 dump_command
                 + (
@@ -184,15 +206,16 @@ def dump_data_sources(
                     '--clean',
                     '--if-exists',
                 )
-                + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+                + (('--host', shlex.quote(hostname)) if hostname else ())
                 + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
                 + (
                     (
                         '--username',
                         shlex.quote(
                             borgmatic.hooks.credential.parse.resolve_credential(
-                                database['username'], config
-                            )
+                                database['username'],
+                                config,
+                            ),
                         ),
                     )
                     if 'username' in database
@@ -215,42 +238,52 @@ def dump_data_sources(
             )
 
             logger.debug(
-                f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}'
+                f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}',
             )
             if dry_run:
                 continue
 
             if dump_format == 'directory':
                 dump.create_parent_directory_for_dump(dump_filename)
-                execute_command(
+                execute_command(  # noqa: S604
                     command,
                     shell=True,
                     environment=environment,
+                    working_directory=borgmatic.config.paths.get_working_directory(config),
                 )
             else:
                 dump.create_named_pipe_for_dump(dump_filename)
                 processes.append(
-                    execute_command(
+                    execute_command(  # noqa: S604
                         command,
                         shell=True,
                         environment=environment,
                         run_to_completion=False,
-                    )
+                        working_directory=borgmatic.config.paths.get_working_directory(config),
+                    ),
                 )
 
     if not dry_run:
-        patterns.append(
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'postgresql_databases', dumps_metadata
+        )
+        borgmatic.hooks.data_source.config.inject_pattern(
+            patterns,
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'postgresql_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return processes
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -258,12 +291,17 @@ def remove_data_source_dumps(
     actually remove anything.
     '''
     dump.remove_data_source_dumps(
-        make_dump_path(borgmatic_runtime_directory), 'PostgreSQL', dry_run
+        make_dump_path(borgmatic_runtime_directory),
+        'PostgreSQL',
+        dry_run,
     )
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -273,12 +311,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -306,16 +348,15 @@ def restore_data_source_dump(
     hostname, port, username, and password.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
-        'restore_hostname', data_source.get('hostname')
+    hostname = database_config.resolve_database_option(
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
+    port = database_config.resolve_database_option(
+        'port', data_source, connection_params, restore=True
     )
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or data_source.get('restore_username', data_source.get('username'))
+        database_config.resolve_database_option(
+            'username', data_source, connection_params, restore=True
         ),
         config,
     )
@@ -324,7 +365,10 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
+        hostname=hostname,
+        port=port,
+        container=data_source.get('container'),
+        label=data_source.get('label'),
     )
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql')
@@ -333,7 +377,7 @@ def restore_data_source_dump(
         psql_command
         + ('--no-password', '--no-psqlrc', '--quiet')
         + (('--host', hostname) if hostname else ())
-        + (('--port', port) if port else ())
+        + (('--port', str(port)) if port else ())
         + (('--username', username) if username else ())
         + (('--dbname', data_source['name']) if not all_databases else ())
         + (
@@ -354,7 +398,7 @@ def restore_data_source_dump(
         + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
         + (('--dbname', data_source['name']) if not all_databases else ())
         + (('--host', hostname) if hostname else ())
-        + (('--port', port) if port else ())
+        + (('--port', str(port)) if port else ())
         + (('--username', username) if username else ())
         + (('--no-owner',) if data_source.get('no_owner', False) else ())
         + (
@@ -366,7 +410,7 @@ def restore_data_source_dump(
         + tuple(
             itertools.chain.from_iterable(('--schema', schema) for schema in data_source['schemas'])
             if data_source.get('schemas')
-            else ()
+            else (),
         )
     )
 
@@ -384,5 +428,10 @@ def restore_data_source_dump(
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout if extract_process else None,
         environment=environment,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+    )
+    execute_command(
+        analyze_command,
+        environment=environment,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
     )
-    execute_command(analyze_command, environment=environment)

borgmatic/hooks/data_source/sqlite.py

@@ -4,6 +4,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.data_source.config
 from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks.data_source import dump
 
@@ -49,32 +50,43 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping SQLite databases{dry_run_label}')
 
     for database in databases:
         database_path = database['path']
+        dumps_metadata.append(
+            borgmatic.actions.restore.Dump(
+                'sqlite_databases', database['name'], label=database.get('label')
+            )
+        )
 
         if database['name'] == 'all':
             logger.warning('The "all" database name has no meaning for SQLite databases')
+
         if not os.path.exists(database_path):
             logger.warning(
-                f'No SQLite database at {database_path}; an empty database will be created and dumped'
+                f'No SQLite database at {database_path}; an empty database will be created and dumped',
             )
 
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        dump_filename = dump.make_data_source_dump_filename(dump_path, database['name'])
+        dump_filename = dump.make_data_source_dump_filename(
+            dump_path, database['name'], label=database.get('label')
+        )
 
         if os.path.exists(dump_filename):
             logger.warning(
-                f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
+                f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}',
             )
             continue
 
         sqlite_command = tuple(
             shlex.quote(part) for part in shlex.split(database.get('sqlite_command') or 'sqlite3')
         )
-        command = sqlite_command + (
+        command = (
+            *sqlite_command,
+            '-bail',
             shlex.quote(database_path),
             '.dump',
             '>',
@@ -82,27 +94,42 @@ def dump_data_sources(
         )
 
         logger.debug(
-            f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
+            f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}',
         )
         if dry_run:
             continue
 
         dump.create_named_pipe_for_dump(dump_filename)
-        processes.append(execute_command(command, shell=True, run_to_completion=False))
+        processes.append(
+            execute_command(  # noqa: S604
+                command,
+                shell=True,
+                run_to_completion=False,
+                working_directory=borgmatic.config.paths.get_working_directory(config),
+            ),
+        )
 
     if not dry_run:
-        patterns.append(
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'sqlite_databases', dumps_metadata
+        )
+        borgmatic.hooks.data_source.config.inject_pattern(
+            patterns,
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'sqlite_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return processes
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -113,7 +140,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -123,12 +153,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -150,7 +184,8 @@ def restore_data_source_dump(
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
     database_path = connection_params['restore_path'] or data_source.get(
-        'restore_path', data_source.get('path')
+        'restore_path',
+        data_source.get('path'),
     )
 
     logger.debug(f'Restoring SQLite database at {database_path}{dry_run_label}')
@@ -167,7 +202,7 @@ def restore_data_source_dump(
         shlex.quote(part)
         for part in shlex.split(data_source.get('sqlite_restore_command') or 'sqlite3')
     )
-    restore_command = sqlite_restore_command + (shlex.quote(database_path),)
+    restore_command = (*sqlite_restore_command, '-bail', shlex.quote(database_path))
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
     execute_command_with_processes(
@@ -175,4 +210,5 @@ def restore_data_source_dump(
         [extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
     )