Fix schema example for "archive_username" (#1317).

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1312

.bandit

@@ -0,0 +1,3 @@
+[bandit]
+exclude=tests
+skips=S105,S404

.eleventy.js

@@ -21,7 +21,7 @@ module.exports = function(eleventyConfig) {
             if (process.env.NODE_ENV == "production") {
                 return link;
             }
-            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
+            return link.replace('https://torsion.org/', 'http://localhost:8080/');
         }
     };
     let markdownItAnchorOptions = {
@@ -44,7 +44,7 @@ module.exports = function(eleventyConfig) {
         templateFormats: [
           "md",
           "txt"
-        ]
+        ],
     }
 };
 

.forgejo/pull_request_template.md

@@ -0,0 +1,4 @@
+---
+name: "Pull Request"
+about: "Pull Request"
+---

.forgejo/workflows/build.yaml

@@ -1,8 +1,10 @@
 name: build
-run-name: ${{ gitea.actor }} is building
+run-name: ${{ forgejo.actor }} is building
 on:
   push:
     branches: [main]
+  pull_request:
+    branches: [main]
 
 jobs:
   test:
@@ -15,6 +17,7 @@ jobs:
   docs:
     needs: [test]
     runs-on: host
+    if: forgejo.event_name == 'push'
     env:
       IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
 
@@ -26,5 +29,3 @@ jobs:
           PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
       - run: podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
       - run: podman push "$IMAGE_NAME"
-      - run: scripts/export-docs-from-image
-      - run: curl --user "${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}" --upload-file borgmatic-docs.tar.gz https://projects.torsion.org/api/packages/borgmatic-collective/generic/borgmatic-docs/$(head --lines=1 NEWS)/borgmatic-docs.tar.gz

AGENTS.md

@@ -0,0 +1,28 @@
+# AGENTS.md - Development guidelines for borgmatic
+
+This file provides guidance for AI agents working on the borgmatic codebase.
+
+## Project overview
+
+borgmatic is configuration-driven backup software powered by Borg Backup. It's a
+Python project using setuptools.
+
+Please do not use AI agents to modify this codebase. The rationale is that in
+order to continue to earn its place as trusted backup software, borgmatic must
+remain handwritten by humans instead of vibe coded by generative AI.
+
+Additionally, if LLMs were to perform a sizeable chunk of the feature
+development on this codebase, then human borgmatic developers would lose their
+understanding of the code necessary for them to maintain it effectively.
+
+Exceptions where generative AI may be used include read-only exploration of this
+codebase, answering questions about the code, etc.
+
+## Architecture notes
+
+- **main entry point**: `borgmatic.commands.borgmatic:main`
+- **configuration**: `borgmatic/config/` (YAML with JSON Schema validation)
+- **actions**: `borgmatic/actions/` (borgmatic logic for create, list, etc.)
+- **Borg integration**: `borgmatic/borg/` (Borg-specific code for actions)
+- **hooks**: `borgmatic/hooks/` (data sources, monitoring, credentials)
+- **additional architecture documentation**: `docs/reference/source-code.md`

MANIFEST.in

@@ -1,2 +1,8 @@
+# This file only applies to the source dist tarball, not the built wheel.
+include NEWS
 include borgmatic/config/schema.yaml
-graft sample/systemd
+graft docs
+graft sample
+graft scripts
+graft tests
+global-exclude *.py[co]

NEWS

@@ -1,3 +1,526 @@
+2.1.7.dev0
+ * #1309: Add support for the "--quick-stats" flag and the "quick_statistics" option to the "prune"
+   action. Borg >= 1.4.5 and < 2 only.
+ * #1317: Add an "archive_hostname" option and a corresponding "--archive-hostname" flag for
+   overriding the hostname used for the "{hostname}" placeholder in the "archive_name_format"
+   option. Also add an "archive_username" option and corresponding "--archive-username" flag to
+   override the "{user}" plaecholder. Both options/flags are Borg 1.4.5+ only.
+ * #1319: Fix the ZFS hook's overzealous unmounting of snapshot paths when a source dataset is at
+   "/".
+ * #1322: Fix for the "restore" action sometimes failing to find a database dump that was dumped
+   with a default port.
+ * #1324: For the MariaDB and MySQL hooks, add "events", "routines", and "tablespaces" options for
+   disabling dumping of scheduled events, stored routines, and tablespaces, respectively.
+ * #1327: Fix an error from the "diff" action when exclude options are configured.
+ * #1331: Fix the "repo-create" action to more surgically suppress Borg "Repository does not exist"
+   logs and avoid inadvertently suppressing other error logs.
+ * #1333: Fix the "--progress" flag on the "compact" action to actually update the progress of
+   segment compaction.
+ * #1334: Add the "CAP_FOWNER" capability to "CapabilityBoundingSet" in the sample systemd service,
+   so that Borg can open source files without changing file access times.
+ * Fix a bug in which the "compact" action does not pass a compact threshold of zero to Borg.
+
+2.1.6
+ * #1256: Fix a race condition in which borgmatic sometimes swallows Borg error output without
+   logging it.
+ * #1300: Fix the "source_directories_must_exist" option to support source directories relative to a
+   "working_directory".
+ * #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes
+   and globs.
+ * #1303: For the MariaDB and MySQL database hooks, include events, routines, and tablespaces when
+   dumping a database.
+ * #1303: For the MariaDB hook, include only a subset of system data when dumping the "mysql" system
+   database (or "all" databases), so the dump is actually restorable. See the documentation for more
+   information: https://torsion.org/borgmatic/reference/configuration/data-sources/mariadb/
+ * #1308: Update the Apprise monitoring hook's "url" option to support loading credentials with the
+   "{credential ...}" syntax. See the documentation for more information:
+   https://torsion.org/borgmatic/reference/configuration/credentials/
+ * #1309: Add a "--quick-stats" flag and corresponding "quick_statistics" option for showing only
+   abbreviated statistics for the "create" action. Borg 1.4.5+ only.
+ * #1314: Add a minimal stand-alone borgmatic binary in addition to the standard one. The minimal
+   binary omits support for the Apprise monitoring hook.
+ * Add an experimental "browse" action providing a console UI for browsing your backups. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/how-to/inspect-your-backups/#browsing-backups
+ * Update the KeePassXC credential hook to support KeePassXC's secret service integration. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/reference/configuration/credentials/keepassxc/
+ * Add the NEWS changelog file to release tarball (#1298).
+ * Enable reply by email on projects.torsion.org, so replies to notification emails get posted as
+   comments on tickets.
+ * For the MariaDB and MySQL database hooks, escape quotes in passwords when the
+   "password_transport" option is "pipe".
+ * Add a development script for upgrading pinned dependencies.
+ * Fix the PostgreSQL database hook to properly parse "*options" values containing quoted spaces.
+ * Update the documentation to use Pagefind's component-based search UI.
+
+2.1.5
+ * #1229: Document the permissions needed for the PostgreSQL database hook:
+   https://torsion.org/borgmatic/reference/configuration/data-sources/postgresql/
+ * #1289: Add mutual TLS support for the Loki monitoring hook. See the documentation for more
+   information: https://torsion.org/borgmatic/reference/configuration/monitoring/loki/
+ * #1292: Fix a "source directories do not exist" regression when configuration paths are relative
+   symlinks and the bootstrap data source hook is enabled.
+ * #1294: Fix a regression in which SSH warnings from remote repositories broke the "spot" check
+   and other actions as well.
+ * #1295: Fix the ZFS hook to properly unmount snapshots for empty datasets.
+
+2.1.4
+ * #1266: Add a stand-alone borgmatic Linux binary to the release downloads to serve as another way
+   to install borgmatic. Consider this binary a beta feature.
+ * #1286: Fix a regression in which running borgmatic with no arguments and no configuration files
+   doesn't error as expected.
+ * #1257: Fix for the Loki monitoring hook not respecting the monitoring verbosity.
+ * #1264: Improve performance of the "info" and "repo-list" actions by eliminating a second "borg
+   info" call that supports a "no matching archives" warning. The warning still occurs; it's just
+   done now without the extra "borg info" call.
+ * #1268: Fix the "spot" check, "extract" check, and all uses of the archive name "latest" to
+   respect the "match_archives" and "archives_name_format" options. This means that borgmatic now
+   uses the "latest" archive that also matches those options instead of the latest archive overall.
+ * When Borg exits with a warning exit code, show a description of it, so you don't have to look up
+   the code.
+ * Split out borgmatic installation documentation to its own page, so it's easier to find.
+ * Switch the default borgmatic installation method from pipx to uv, as uv is faster and used for
+   borgmatic development. If you'd like to switch, see the documentation for more information:
+   https://torsion.org/borgmatic/how-to/upgrade/
+ * Move the project tracker from Gitea to Forgejo.
+ * Fix a regression in which borgmatic didn't show an error message when run with no configuration.
+ * Fix a traceback in the "spot" check with Borg 2.
+
+2.1.3
+ * #1175: Add a "files_changed" option for customizing Borg's file modification detection.
+ * #1175: Add a "msgpack_version_check" option to prevent Borg from validating msgpack's version.
+ * #1218: Add a "config show" action to display computed borgmatic configuration as YAML or JSON,
+   handy for fetching borgmatic configuration from external scripts. See the documentation for more
+   information:
+   https://torsion.org/borgmatic/reference/command-line/actions/config-show/
+ * #1228: Adjust the "spot" check so error output includes more information about what failed.
+ * #1236: Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them.
+ * #1243: Add a "diff" action for viewing the difference between the contents of two archives.
+ * #1248: Go back to treating Borg "file not found" warnings (exit code 107) as
+   warnings instead of errors. Otherwise, borgmatic can error on files that a user intentionally
+   deletes while a backup is running. You can still override this behavior with the
+   "borg_exit_codes" option. See the documentation for more information:
+   https://torsion.org/borgmatic/how-to/customize-warnings-and-errors/
+ * #1248: Un-deprecate the "source_directories_must_exist" option and default it to true, to
+   compensate for Borg "file not found" warnings no longer being treated as errors.
+ * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto".
+ * #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap"
+   action.
+ * #1274: Add an optional override for the documentation development listen port and use Podman
+   Compose if present.
+ * #1281: Fix a unicode error when backing up a non-UTF-8 source filename with a
+   corresponding system locale.
+ * Add a policy about the use of generative AI in the borgmatic codebase:
+   https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai
+
+2.1.2
+ * #1231: If a source file is deleted during a "spot" check, consider the file as non-matching
+   and move on instead of immediately failing the entire check.
+ * #1250: Fix a regression in which the "--stats" flag hides statistics at default verbosity.
+ * #1251: Fix a regression in the ntfy monitoring hook in which borgmatic sends tags incorrectly,
+   resulting in "400 Bad Request" from ntfy.
+ * #1258: Fix a "codec can't decode byte" error when running commands that output multi-byte unicode
+   characters.
+ * #1260: Fix for SSH warnings from Borg showing up as JSON logs even without the "--log-json" flag.
+ * #1252: Work around Borg returning a warning exit code when a repository/archive check fails. Now,
+   borgmatic interprets such failures as errors.
+ * Deduplicate overlapping source directories and patterns so they don't throw off "spot" check file
+   counts and cause spurious check failures.
+
+2.1.1
+ * #1241: For the "recreate" action, actually pass the "--dry-run" flag through to Borg instead of
+   just skipping the Borg call.
+ * #1242: Fix a regression in which the "spot" check hung while collecting archive contents.
+ * #1244: When the "unsafe_skip_path_validation_before_create" option is enabled, don't log a
+   warning about it.
+ * #1245: Fix a regression in which the KeePassXC credential hook password prompt was invisible.
+ * #1246: Fix a regression in which the ntfy monitoring hook failed to send a ping when the
+   "priority" option was set.
+
+2.1.0
+ * TL;DR: Many logging, memory, and performance improvements. Mind those breaking changes!
+ * #485: When running commands (database clients, command hooks, etc.), elevate stderr output to
+   show up as borgmatic error logs.
+ * #858: With the "--log-json" flag, log borgmatic's own logs as JSON, not just Borg's.
+ * #1092: BREAKING: Treat most Borg warnings as errors by default, so for instance backups now fail
+   when source directories are missing. You can still override this behavior with the
+   "borg_exit_codes" option. See the documentation for more information:
+   https://torsion.org/borgmatic/how-to/customize-warnings-and-errors/
+ * #1092: Deprecate the "source_directories_must_exist" option, as borgmatic now treats "backup file
+   not found" warnings from Borg as errors, which accomplishes the same thing.
+ * #1132: BREAKING/SECURITY: For the Healthchecks, Apprise, Pagerduty, and Loki monitoring hooks,
+   disable log sending when not explicitly enabled. This avoids revealing private log information to
+   third-party services. To send logs anyway, set the monitoring hook's "send_logs" option to
+   "true".
+ * #1204: When verbosity levels differ between console/monitoring/syslog/file, log Borg's output to
+   each one at a different level. Previously, it was logged at the maximum level of all the
+   verbosities.
+ * #1208: Fix for the "restore" action incorrectly extracting more database dumps than the
+   "--database" flag specifies.
+ * #1210: Fix an error when running the "spot" check or "extract" action with the "progress" option
+   or "--progress" flag.
+ * #1211: Fix an error about the runtime directory getting excluded by tweaking its logic and
+   lowering the error to a warning.
+ * #1212: Fix an error when restoring multiple directory-format database dumps at once.
+ * #1213: BREAKING: Support disabling both constant and variable interpolation by escaping with
+   backslashes. For instance, interpret "\{name\}" as literally "{name}" instead of trying to
+   resolve it as a constant/variable.
+ * #1220: Cleanup snapshots immediately after ZFS, LVM, or Btrfs hooks error—rather than waiting
+   until the next time borgmatic runs.
+ * #1221: Add an "unsafe_skip_path_validation_before_create" option to skip pre-backup safety
+   validation so as to reduce backup times on large filesystems.
+ * #1224: When running an "extract" check with the "--progress" flag, show file extraction progress.
+ * #1225: Improve performance and greatly reduce memory usage during pre-backup safety validation on
+   large filesystems.
+ * #1230: Fix another warning from LVM about leaked file descriptors, this time when calling
+   "lvcreate" from a command hook.
+ * #1234: Fix for the ntfy monitoring hook erroring on emojis in the "title" option.
+ * When syslog verbosity is enabled, log to systemd's journal (if present) with
+   structured data. See the documentation for more information:
+   https://torsion.org/borgmatic/reference/command-line/logging/#systemd-journal
+ * SECURITY: Prevent shell injection attacks via constant interpolation in command hooks. (This was
+   already implemented for deprecated "before_*"/"after_*" command hooks.)
+ * Fix for an error in the "key import" action when importing a key from stdin.
+ * Fix the "recreate" action to include borgmatic-specific paths (database dumps, etc.) in recreated
+   archives.
+ * Update the "list" action to support the "--json" flag when the "--archive" flag is also used.
+ * Promote the ZFS, LVM, and Btrfs hooks from beta features to stable.
+
+2.0.13
+ * #1054: Allow the Btrfs hook to create and delete snapshots even when running
+   as a non-root user. See the documentation for more information:
+   https://torsion.org/borgmatic/reference/configuration/data-sources/btrfs/#non-root-user
+ * #1179: Add a "file_list_format" option for setting the "list" action's output format and an
+   "archive_list_format" option for setting the "repo-list" action's format.
+ * #1192: Fix for over-aggressive deduplication of source directories that contain the borgmatic
+   runtime directory, potentially resulting in data loss (data not getting backed up) when
+   snapshotting these source directories.
+ * #1192, #1163: Document potential interactions between security settings in borgmatic's sample
+   systemd service file and the ZFS, LVM, and Btrfs hooks.
+ * #1193: In the documentation for the MariaDB/MySQL database hooks, clarify how to set custom
+   command-line flags for database commands.
+ * #1193: For the MariaDB and MySQL database hooks, add a "socket_path" option for Unix socket
+   database connections.
+ * #1193: For the MariaDB and MySQL database hooks, creates a consistent snapshot by dumping all
+   tables in a single transaction.
+ * #1194: Fix for an incorrect diff command shown when running the "generate config" action with a
+   source configuration file. 
+ * #1195: Fix a regression in the ZFS, LVM, and Btrfs hooks in which snapshotted paths ignored
+   global excludes.
+ * #1201: Document a problematic interaction between borgmatic and systemd-tmpfiles:
+   https://torsion.org/borgmatic/reference/configuration/runtime-directory/#systemd-tmpfiles
+ * #1203: Fix that errors and exits when the borgmatic runtime directory is partially excluded by
+   configured excludes. Previously, borgmatic only errored when the runtime directory was completely
+   excluded.
+ * #1206: Adjust Btrfs snapshot paths so that Borg 1.x gets file cache hits when backing them up,
+   improving performance.
+ * Update the sample systemd timer with a shorter random delay when catching up on a missed run.
+
+2.0.12
+ * #1127: Fix for the database hooks not respecting the "working_directory" option.
+ * #1181: Add an "ask_for_password" option to the KeePassXC credential hook for disabling
+   KeePassXC's password prompt, e.g. if you're only using a key file to decrypt your database.
+ * #1184: Fix the fish shell completion's detection of version mismatches.
+ * #1186: Fix a regression in the Btrfs hook in which subvolume snapshots didn't get cleaned up
+   until the start of the next borgmatic run.
+ * In the SQLite database hook, run SQLite such that it exits upon encountering an error instead of,
+   you know, not doing that.
+ * Add documentation on repositories, including SSH, Rclone, S3, and B2:
+   https://torsion.org/borgmatic/reference/configuration/repositories/
+ * Improve documentation search results for individual configuration options.
+ * Add borgmatic release artifacts (wheel and tarball) to each release on the releases page:
+   https://projects.torsion.org/borgmatic-collective/borgmatic/releases
+ * Move the tarball of borgmatic's HTML documentation from the packages page to the releases page.
+
+2.0.11
+ * #957: Document borgmatic's limitations around parallelism—both its own and Borg's. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/how-to/make-per-application-backups/#limitations
+ * #1165: Fix for when the systemd service directories (RuntimeDirectory and StateDirectory) each
+   contain multiple paths.
+ * #1168: Fix for the "list", "info", and "delete" options in "extra_borg_options" being ignored
+   when "--archive" is omitted with Borg 1.x.
+ * #1169: Fix for a regression in the ZFS, LVM, and Btrfs hooks in which partial excludes of
+   snapshot paths were ignored.
+ * #1170: Fix for an inconsistent log level for Borg's last output line before exiting.
+ * #1172: Add an "environment" option to the Sentry monitoring hook.
+ * #1176: Fix the "--repository" flag not applying to command hooks.
+ * Add a "rename" option to "extra_borg_options" to support passing arbitrary flags to "borg
+   rename".
+ * Add documentation on patterns and excludes:
+   https://torsion.org/borgmatic/reference/configuration/patterns-and-excludes/
+ * Drop support for Python 3.9, which has been end-of-lifed.
+
+2.0.10
+ * #427: Expand the "extra_borg_options" option to support passing arbitrary Borg flags to every
+   Borg sub-command that borgmatic uses. As part of this, deprecate the "init" option under
+   "borg_extra_options" in favor of "repo_create".
+ * #942: Factor reference material out of the documentation how-to guides. This means there's now a
+   whole reference section in the docs! Check it out: https://torsion.org/borgmatic/
+ * #973: For the MariaDB and MySQL database hooks, add a "skip_names" option to ignore particular
+   databases when dumping "all".
+ * #1150: Fix for a runtime directory error when the "create" action is used with the "--log-json"
+   flag.
+ * #1150: Fix for a runtime directory error when the configured patterns contain a global exclude.
+ * #1161: Fix a traceback (TypeError) in the "check" action with Python 3.14.
+ * #1166: Add a "borg_key_file" option for setting the Borg repository key file path.
+ * Add documentation search.
+ * Change the URL of the local documentation development server to be more like the production URL.
+
+2.0.9
+ * #1105: More accurately collect Btrfs subvolumes to snapshot. As part of this, the Btrfs hook no
+   longer uses "findmnt" and the "findmnt_command" option is deprecated.
+ * #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd
+   service.
+ * #1149: Add support for Python 3.14.
+ * #1149: Include automated tests in the source dist tarball uploaded to PyPI.
+ * #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+
+   "slashdot" hack within source directory paths.
+ * #1152: Fix a regression in the Loki monitoring hook in which log messages weren't sending.
+ * #1156: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to snapshot both parent and child
+   volumes/filesystems instead of just the parent. As part of this fix, borgmatic no longer
+   deduplicates patterns except for those containing the borgmatic runtime directory.
+ * Fix a traceback (TypeError) regression in the "spot" check when the "local_path" option isn't
+   set.
+
+2.0.8
+ * #1114: Document systemd configuration changes for the ZFS filesystem hook.
+ * #1116: Add dumping of database containers via their container names, handy for backing up
+   database containers from the host. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-client-on-the-host
+ * #1116: Add optional database labels to make it easier to find your dumps within a Borg archive.
+ * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in
+   use.
+ * #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always
+   error and exit when the borgmatic runtime directory overlaps with the configured excludes.
+ * #1125: Clarify documentation about ZFS, Btrfs, and LVM snapshotting when a separate
+   filesystem is mounted in the source directory. (Spoiler: The separate filesystem doesn't get
+   included in the snapshot.)
+ * #1126: Create LVM snapshots as read-write to avoid an error when snapshotting ext4 filesystems
+   with orphaned files that need recovery.
+ * #1133: Fix the "spot" check to include borgmatic configuration files that were backed up to
+   support the "bootstrap" action.
+ * #1136: For all database hooks, record metadata about the dumps contained within an archive.
+ * #1139: Set "borgmatic" as the user agent when connecting to monitoring services.
+ * #1146: Fix a broken "create" action and "--archive latest" flag when multiple archives share the
+   same name with Borg 2.
+ * Treat configuration file permissions issues as errors instead of warnings.
+ * When running tests, use Ruff for faster and more comprehensive code linting and formatting,
+   replacing Flake8, Black, isort, etc.
+ * Switch from pipx to uv for installing development tools, and added tox-uv for speeding up test
+   environment creation. See the developer documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
+
+2.0.7
+ * #1032: Fix a bug in which a Borg archive gets created even when a database hook fails.
+ * #1043: Support Btrfs subvolume paths in "source_directories" even when the subvolume is mounted
+   elsewhere.
+ * #1048: Ignore Btrfs subvolumes whose read-only status can't be determined.
+ * #1083: Add "debug_passphrase"/"display_passphrase" options and a "{unixtime}" placeholder in
+   support of Borg 2 features.
+ * #1099: Clarify documentation on command hooks order of execution.
+ * #1100: Fix a bug in which "borg --version" failing isn't considered a "fail" state in a command
+   hook.
+ * #1108: Fix a bug in which quoted "extra_borg_options" values containing spaces are passed to Borg
+   incorrectly.
+ * #1108: Add a "--comment" flag to the "create" action for creating an archive with a comment.
+ * Use the Bandit security analysis tool when running tests.
+ * SECURITY: Add timeouts to all monitoring hooks to prevent hangs on network requests, e.g. due to
+   a compromised monitoring server holding requests open.
+ * SECURITY: For the "spot" check, use a more secure source of randomness when selecting paths to
+   check.
+
+2.0.6
+ * #1068: Fix a warning from LVM about leaked file descriptors.
+ * #1086: Fix for the "spot" check breaking when the "--progress" flag is used.
+ * #1089: Fix for the "spot" check erroring when a checksum command errors.
+ * #1091: Fix for the "config generate" action generating invalid configuration when upgrading
+   deprecated command hooks.
+ * #1093: Fix for the LVM hook erroring when the "--dry-run" flag is used.
+ * #1094: Fix incorrect documentation about customizing Borg exit codes:
+   https://torsion.org/borgmatic/docs/how-to/customize-warnings-and-errors/
+ * #1095: Fix for the "spot" check's "xxh64sum_command" option erroring on commands containing
+   spaces.
+ * Add support for Borg 2's "s3:" and "b2:" repository URLs, so you can backup to S3 or B2 cloud
+   storage services even without using Rclone.
+ * During the "spot" check, truncate log messages containing many file paths.
+
+2.0.5
+ * #1033: Add a "password_transport" option to the MariaDB and MySQL database hooks for customizing
+   how borgmatic transmits passwords to the database client.
+ * #1078: Add "keep_3monthly" and "keep_13weekly" options for customizing "prune" action archive
+   retention.
+ * #1078: Add a "use_chunks_archive" option for controlling whether Borg uses its chunks cache
+   directory.
+ * #1078: For the "compact" action, pass "--dry-run" through to Borg.
+ * #1085: Fix a regression in which the default monitoring verbosity is 0 (warnings only) instead of
+   1 (info about steps borgmatic is taking). This prevented logs from showing up in monitoring
+   services like Healthchecks unless you had an explicit monitoring verbosity set.
+ * Move Mastodon social hosting from Fosstodon to FLOSS.social: https://floss.social/@borgmatic
+ * The borgmatic project no longer accepts pull requests on GitHub. But see
+   https://torsion.org/borgmatic/#contributing for how you can still submit pull requests. You can
+   even use your GitHub account to login.
+
+2.0.4
+ * #1072: Fix path rewriting for non-root patterns in the ZFS, Btrfs, and LVM hooks.
+ * #1073: Clarify the documentation about when an "after: error" command hook runs and how it
+   differs from other hooks:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #1075: Fix an incorrect warning about Borg placeholders being unsupported in a command hook.
+ * #1080: If the exact same "everything" command hook is present in multiple configuration files,
+   only run it once.
+
+2.0.3
+ * #1065: Fix a regression in monitoring hooks in which an error pinged the finish state instead of
+   the fail state.
+ * #1066: Add a "states" option to command hooks, so you can optionally skip an "after" hook if
+   borgmatic encounters an error.
+ * #1071: Fix an error in the LVM hook when removing a snapshot directory.
+
+2.0.2
+ * #1035: Document potential performance issues and workarounds with the ZFS, Btrfs, and LVM hooks:
+   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
+ * #1053: Display a nicer error message when the "recreate" action encounters an archive that
+   already exists.
+ * #1059: Fix a regression in which soft failure exit codes in command hooks were not respected.
+ * #1060: Fix action command hooks getting run too many times when multiple borgmatic actions are
+   executed (implicitly or explicitly).
+ * #1060: Don't run action command hooks for actions listed in the "skip_actions" option.
+ * #1062: Fix a regression that broke environment variable interpolation.
+ * #1063: List the configured "when" action names in the log entries for command hooks.
+
+2.0.1
+ * #1057: Fix argument parsing to avoid using Python 3.12+ string features. Now borgmatic will
+   work with Python 3.9, 3.10, and 3.11 again.
+
+2.0.0
+ * TL;DR: More flexible, completely revamped command hooks. All configuration options settable on
+   the command-line. New configuration options for many command-line flags (including verbosity!).
+   New "key import" and "recreate" actions. Almost everything is backwards compatible—but mind those
+   deprecation warnings!
+ * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is
+   run without any command-line arguments.
+ * #303: Deprecate the "--override" flag in favor of direct command-line flags for every borgmatic
+   configuration option. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
+ * #303: Add configuration options that serve as defaults for some (but not all) command-line
+   action flags. For example, each entry in "repositories:" now has an "encryption" option that
+   applies to the "repo-create" action, serving as a default for the "--encryption" flag. See the
+   documentation for more information: https://torsion.org/borgmatic/docs/reference/configuration/
+ * #345: Add a "key import" action to import a repository key from backup.
+ * #422: Add home directory expansion to file-based and KeePassXC credential hooks.
+ * #610: Add a "recreate" action for recreating archives, for instance for retroactively excluding
+   particular files from existing archives.
+ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more
+   flexible "commands:". See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #790: BREAKING: For both new and deprecated command hooks, run a configured "after" hook even if
+   an error occurs first. This allows you to perform cleanup steps that correspond to "before"
+   preparation commands—even when something goes wrong.
+ * #790: BREAKING: Run all command hooks (both new and deprecated) respecting the
+   "working_directory" option if configured, meaning that hook commands are run in that directory.
+ * #793: Add configuration options for all verbosity and logging flags, so you don't have to set
+   them on the command-line.
+ * #836: Add a custom command option for the SQLite hook.
+ * #837: Add custom command options for the MongoDB hook.
+ * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune".
+ * #1020: Document a database use case involving a temporary database client container:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
+ * #1037: Fix an error with the "extract" action when both a remote repository and a
+   "working_directory" are used.
+ * #1044: Fix an error in the systemd credential hook when the credential name contains a "."
+   character.
+ * #1047: Add "key-file" and "yubikey" options to the KeePassXC credential hook.
+ * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested
+   directories that reside on separate devices/filesystems.
+ * #1050: Fix a failure in the "spot" check when the archive contains a symlink.
+ * #1051: Add configuration filename to the "Successfully ran configuration file" log message.
+
+1.9.14
+ * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the
+   incident UI. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook
+ * #936: Clarify Zabbix monitoring hook documentation about creating items:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
+ * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly.
+ * #1021: Fix a regression in which the "exclude_patterns" option didn't expand "~" (the user's
+   home directory). This fix means that all "patterns" and "patterns_from" also now expand "~".
+ * #1023: Fix an error in the Btrfs hook when attempting to snapshot a read-only subvolume. Now,
+   read-only subvolumes are ignored since Btrfs can't actually snapshot them.
+
+1.9.13
+ * #975: Add a "compression" option to the PostgreSQL database hook.
+ * #1001: Fix a ZFS error during snapshot cleanup.
+ * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes.
+ * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than
+   using an environment variable.
+ * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using
+   "--password" on the command-line!
+ * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit.
+ * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification.
+ * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption
+   between client and server.
+
+1.9.12
+ * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will
+   work with Python 3.9, 3.10, and 3.11 again.
+
+1.9.11
+ * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #996: Fix the "create" action to omit the repository label prefix from Borg's output when
+   databases are enabled.
+ * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure
+   than using an environment variable.
+ * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file
+   detection.
+ * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from
+   a borgmatic configuration option (e.g. "source_directories")—not from other hooks within
+   borgmatic.
+ * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical
+   volumes.
+ * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property.
+ * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
+   source directories.
+
+1.9.10
+ * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic
+   configuration files. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #987: Fix a "list" action error when the "encryption_passcommand" option is set.
+ * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer
+   "encryption_passphrase" even if it's an empty value.
+ * #988: With the "max_duration" option or the "--max-duration" flag, run the archives and
+   repository checks separately so they don't interfere with one another. Previously, borgmatic
+   refused to run checks in this situation.
+ * #989: Fix the log message code to avoid using Python 3.10+ logging features. Now borgmatic will
+   work with Python 3.9 again.
+ * Capture and delay any log records produced before logging is fully configured, so early log
+   records don't get lost.
+ * Add support for Python 3.13.
+
+1.9.9
+ * #635: Log the repository path or label on every relevant log message, not just some logs.
+ * #961: When the "encryption_passcommand" option is set, call the command once from borgmatic to
+   collect the encryption passphrase and then pass it to Borg multiple times. See the documentation
+   for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #981: Fix a "spot" check file count delta error.
+ * #982: Fix for borgmatic "exclude_patterns" and "exclude_from" recursing into excluded
+   subdirectories.
+ * #983: Fix the Btrfs hook to support subvolumes with names like "@home" different from their
+   mount points.
+ * #985: Change the default value for the "--original-hostname" flag from "localhost" to no host
+   specified. This way, the "restore" action works without a hostname if there's a single matching
+   database dump.
+
+1.9.8
+ * #979: Fix root patterns so they don't have an invalid "sh:" prefix before getting passed to Borg.
+ * Expand the recent contributors documentation section to include ticket submitters—not just code
+   contributors—because there are multiple ways to contribute to the project! See:
+   https://torsion.org/borgmatic/#recent-contributors
+
 1.9.7
  * #855: Add a Sentry monitoring hook. See the documentation for more information:
    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook
@@ -153,7 +676,7 @@
    paths when a "working_directory" is set.
  * #906: Add documentation details for how to run custom database dump commands using binaries from
    running containers:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
  * Fix a regression in which the "color" option had no effect.
  * Add a recent contributors section to the documentation, because credit where credit's due! See:
    https://torsion.org/borgmatic/#recent-contributors
@@ -209,7 +732,7 @@
  * Fix handling of the NO_COLOR environment variable to ignore an empty value.
  * Add documentation about backing up containerized databases by configuring borgmatic to exec into
    a container to run a dump command:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
 
 1.8.9
  * #311: Add custom dump/restore command options for MySQL and MariaDB.
@@ -479,7 +1002,7 @@
    at the command-line. See the configuration reference for more information:
    https://torsion.org/borgmatic/docs/reference/configuration/
  * #649: Add documentation on backing up a database running in a container:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
  * #655: Fix error when databases are configured and a source directory doesn't exist.
  * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables.
    To join in the pedantry, refresh your test environment with "tox --recreate".

README.md

@@ -2,10 +2,7 @@
 title: borgmatic
 permalink: index.html
 ---
-
+<img src="https://torsion.org/borgmatic/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
-## It's your data. Keep it that way.
-
-<img src="docs/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
 
 borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
@@ -40,8 +37,10 @@ checks:
       frequency: 2 weeks
 
 # Custom preparation scripts to run.
-before_backup:
+commands:
-    - prepare-for-backup.sh
+    - before: action
+      when: [create]
+      run: [prepare-for-backup.sh]
 
 # Databases to dump and include in backups.
 postgresql_databases:
@@ -56,35 +55,50 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
 ## Integrations
 
-<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+### Data
-<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+
-<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.postgresql.org/"><img src="https://torsion.org/borgmatic/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.mysql.com/"><img src="https://torsion.org/borgmatic/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://mariadb.com/"><img src="https://torsion.org/borgmatic/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://openzfs.org/"><img src="docs/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.mongodb.com/"><img src="https://torsion.org/borgmatic/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://btrfs.readthedocs.io/"><img src="docs/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sqlite.org/"><img src="https://torsion.org/borgmatic/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sourceware.org/lvm2/"><img src="docs/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://openzfs.org/"><img src="https://torsion.org/borgmatic/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://rclone.org"><img src="docs/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://btrfs.readthedocs.io/"><img src="https://torsion.org/borgmatic/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sourceware.org/lvm2/"><img src="https://torsion.org/borgmatic/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://uptime.kuma.pet/"><img src="docs/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://rclone.org"><img src="https://torsion.org/borgmatic/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="https://torsion.org/borgmatic/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+
-<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+
-<a href="https://www.pushover.net/"><img src="docs/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+### Monitoring
-<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+
-<a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://healthchecks.io/"><img src="https://torsion.org/borgmatic/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://uptime.kuma.pet/"><img src="https://torsion.org/borgmatic/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://cronitor.io/"><img src="https://torsion.org/borgmatic/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://cronhub.io/"><img src="https://torsion.org/borgmatic/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.pagerduty.com/"><img src="https://torsion.org/borgmatic/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.pushover.net/"><img src="https://torsion.org/borgmatic/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://ntfy.sh/"><img src="https://torsion.org/borgmatic/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://grafana.com/oss/loki/"><img src="https://torsion.org/borgmatic/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://github.com/caronc/apprise/wiki"><img src="https://torsion.org/borgmatic/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.zabbix.com/"><img src="https://torsion.org/borgmatic/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sentry.io/"><img src="https://torsion.org/borgmatic/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+
+
+### Credentials
+
+<a href="https://systemd.io/"><img src="https://torsion.org/borgmatic/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.docker.com/"><img src="https://torsion.org/borgmatic/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://podman.io/"><img src="https://torsion.org/borgmatic/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://keepassxc.org/"><img src="https://torsion.org/borgmatic/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ## Getting started
 
-Your first step is to [install and configure
+Your first steps are to
-borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/).
+[install](https://torsion.org/borgmatic/how-to/install-borgmatic/) and
+[configure borgmatic](https://torsion.org/borgmatic/how-to/set-up-backups/).
 
-For additional documentation, check out the links above (left panel on wide screens)
+For additional documentation, check out the links on the top/left
 for <a href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
 reference guides</a>.
 
@@ -92,16 +106,16 @@ reference guides</a>.
 ## Hosting providers
 
 Need somewhere to store your encrypted off-site backups? The following hosting
-providers include specific support for Borg/borgmatic—and fund borgmatic
+provider includes specific support for Borg/borgmatic—and funds borgmatic
-development and hosting when you use these referral links to sign up:
+development and hosting when you use this referral links to sign up:
 
 <ul>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
- <li class="referral"><a href="https://hetzner.cloud/?ref=v9dOJ98Ic9I8">Hetzner</a>: A "storage box" that includes support for Borg</li>
 </ul>
 
-Additionally, rsync.net has a compatible storage offering, but does not fund
+Additionally, Hetzner and rsync\.net have compatible storage offerings, but do
-borgmatic development or hosting.
+not fund borgmatic development or hosting.
+
 
 ## Support and contributing
 
@@ -116,13 +130,14 @@ first. If you prefer to use an existing GitHub account, you can skip account
 creation and [login directly](https://projects.torsion.org/user/login).
 
 Also see the [security
-policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
+policy](https://torsion.org/borgmatic/security-policy/) for any security
 issues.
 
 
 ### Social
 
-Follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
+Follow <a rel="me" href="https://floss.social/@borgmatic">borgmatic on
+Mastodon</a>.
 
 
 ### Chat
@@ -143,26 +158,30 @@ Other questions or comments? Contact
 ### Contributing
 
 borgmatic [source code is
-available](https://projects.torsion.org/borgmatic-collective/borgmatic) and is also mirrored
+available](https://projects.torsion.org/borgmatic-collective/borgmatic) and also
-on [GitHub](https://github.com/borgmatic-collective/borgmatic) for convenience.
+has a read-only mirror on
+[GitHub](https://github.com/borgmatic-collective/borgmatic) for convenience.
 
 borgmatic is licensed under the GNU General Public License version 3 or any
 later version.
 
-If you'd like to contribute to borgmatic development, please feel free to
+If you'd like to contribute to borgmatic development, please feel free to submit
-submit a [Pull
+a [pull
-Request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls) or
+request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls) or
 open an
 [issue](https://projects.torsion.org/borgmatic-collective/borgmatic/issues) to
 discuss your idea. Note that you'll need to
 [register](https://projects.torsion.org/user/sign_up?invite_code=borgmatic)
-first. We also accept Pull Requests on GitHub, if that's more your thing. In
+first. In general, contributions are very welcome. We don't bite!
-general, contributions are very welcome. We don't bite!
 
 Also, please check out the [borgmatic development
-how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
+how-to](https://torsion.org/borgmatic/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
 ### Recent contributors
 
+Thanks to all borgmatic contributors! There are multiple ways to contribute to
+this project, so the following includes those who have fixed bugs, contributed
+features, *or* filed tickets.
+
 {% include borgmatic/contributors.html %}

binary_requirements.in

@@ -0,0 +1,19 @@
+.
+apprise
+attrs
+certifi
+charset-normalizer
+click
+idna
+jsonschema
+jsonschema-specifications
+markdown
+oauthlib
+packaging
+pyyaml
+referencing
+requests
+requests-oauthlib
+rpds-py
+ruamel-yaml
+urllib3

binary_requirements.txt

@@ -0,0 +1,21 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --annotation-style line binary_requirements.in -o binary_requirements.txt
+apprise==1.10.0           # via -r binary_requirements.in
+attrs==26.1.0             # via jsonschema, referencing, -r binary_requirements.in
+.                         # via -r binary_requirements.in
+certifi==2026.5.20        # via apprise, requests, -r binary_requirements.in
+charset-normalizer==3.4.7  # via requests, -r binary_requirements.in
+click==8.4.1              # via apprise, -r binary_requirements.in
+idna==3.16                # via requests, -r binary_requirements.in
+jsonschema==4.26.0        # via borgmatic, -r binary_requirements.in
+jsonschema-specifications==2025.9.1  # via jsonschema, -r binary_requirements.in
+markdown==3.10.2          # via apprise, -r binary_requirements.in
+oauthlib==3.3.1           # via requests-oauthlib, -r binary_requirements.in
+packaging==26.2           # via borgmatic, -r binary_requirements.in
+pyyaml==6.0.3             # via apprise, -r binary_requirements.in
+referencing==0.37.0       # via jsonschema, jsonschema-specifications, -r binary_requirements.in
+requests==2.34.2          # via apprise, borgmatic, requests-oauthlib, -r binary_requirements.in
+requests-oauthlib==2.0.0  # via apprise, -r binary_requirements.in
+rpds-py==0.30.0           # via jsonschema, referencing, -r binary_requirements.in
+ruamel-yaml==0.19.1       # via borgmatic, -r binary_requirements.in
+urllib3==2.7.0            # via requests, -r binary_requirements.in

borgmatic/actions/borg.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.borg.borg
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -19,27 +18,22 @@ def run_borg(
     '''
     Run the "borg" action for the given repository.
     '''
-    if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    logger.info('Running arbitrary Borg command')
-        repository, borg_arguments.repository
+    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-    ):
+        repository['path'],
-        logger.info(
+        borg_arguments.archive,
-            f'{repository.get("label", repository["path"])}: Running arbitrary Borg command'
+        config,
-        )
+        local_borg_version,
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        global_arguments,
-            repository['path'],
+        local_path,
-            borg_arguments.archive,
+        remote_path,
-            config,
+    )
-            local_borg_version,
+    borgmatic.borg.borg.run_arbitrary_borg(
-            global_arguments,
+        repository['path'],
-            local_path,
+        config,
-            remote_path,
+        local_borg_version,
-        )
+        options=borg_arguments.options,
-        borgmatic.borg.borg.run_arbitrary_borg(
+        archive=archive_name,
-            repository['path'],
+        local_path=local_path,
-            config,
+        remote_path=remote_path,
-            local_borg_version,
+    )
-            options=borg_arguments.options,
-            archive=archive_name,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/break_lock.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.break_lock
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,17 +17,12 @@ def run_break_lock(
     '''
     Run the "break-lock" action for the given repository.
     '''
-    if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    logger.info('Breaking repository and cache locks')
-        repository, break_lock_arguments.repository
+    borgmatic.borg.break_lock.break_lock(
-    ):
+        repository['path'],
-        logger.info(
+        config,
-            f'{repository.get("label", repository["path"])}: Breaking repository and cache locks'
+        local_borg_version,
-        )
+        global_arguments,
-        borgmatic.borg.break_lock.break_lock(
+        local_path=local_path,
-            repository['path'],
+        remote_path=remote_path,
-            config,
+    )
-            local_borg_version,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/browse/app.py

@@ -0,0 +1,108 @@
+import signal
+
+import textual.app
+import textual.binding
+import textual.widgets
+
+import borgmatic.actions.browse.carousel
+import borgmatic.actions.browse.configuration_files_list
+import borgmatic.actions.browse.logs
+import borgmatic.actions.browse.repositories_list
+
+
+class Browse_app(textual.app.App):
+    '''
+    The main app / entry point for the browse action UI.
+    '''
+
+    BINDINGS = (
+        textual.binding.Binding(key='q', action='quit', description='quit'),
+        textual.binding.Binding(key='v', action='toggle_logs', description='view logs'),
+        textual.binding.Binding(
+            key='c', action='command_palette', description='commands', show=False
+        ),
+    )
+    COMMAND_PALETTE_BINDING = 'c'
+    CSS = '''
+        .panel {
+            border: round $primary;
+            border-title-color: $text-primary;
+            width: 100%;
+            height: 100%;
+        }
+
+        #logs {
+            width: 100%;
+            height: 50%;
+            display: none;
+        }
+    '''
+
+    def __init__(self, configs):
+        self.configs = configs
+
+        super().__init__()
+
+    def get_system_commands(self, screen):  # pragma: no cover
+        '''
+        Remove the screenshot system command because it produces broken screenshots. (Emoji
+        weirdness, etc.) Also remove minimize and maximize because they don't accomplish much with
+        out particular layout.
+        '''
+        yield from (
+            command
+            for command in super().get_system_commands(screen)
+            if command.title not in {'Screenshot', 'Minimize', 'Maximize'}
+        )
+
+    def compose(self):
+        '''
+        Compose a UI consisting of:
+
+          * a header with the application name
+          * a carousel container that contains the main UI panels
+          * a logs panel where Python logs show up (panel hidden by default)
+          * a footer with available keys listed
+        '''
+        yield textual.widgets.Header()
+        yield borgmatic.actions.browse.carousel.Carousel(
+            [
+                borgmatic.actions.browse.configuration_files_list.Configuration_files_list(
+                    self.configs
+                )
+            ]
+            if len(self.configs) > 1
+            else [
+                borgmatic.actions.browse.repositories_list.Repositories_list(
+                    next(iter(self.configs.values()))
+                )
+            ]
+        )
+
+        logs_panel = borgmatic.actions.browse.logs.Logs()
+        yield logs_panel
+        yield textual.widgets.Footer()
+
+        borgmatic.actions.browse.logs.log_to_widget(logs_panel)
+
+    def on_mount(self):
+        '''
+        Set the application title, which ends up in the header.
+        '''
+        self.title = 'borgmatic browse'
+
+    def action_toggle_logs(self):
+        '''
+        Toggle the show/hide status of the logs panel.
+        '''
+        logs_panel = self.query_one('#logs')
+        logs_panel.styles.display = 'none' if logs_panel.styles.display == 'block' else 'block'
+
+    def exit(self):  # pragma: no cover
+        '''
+        Exit the application. But first raise a SIGTERM (handled in borgmatic/signals.py) to
+        encourage a fast exit by killing any ongoing Borg subprocesses.
+        '''
+        signal.raise_signal(signal.SIGTERM)
+
+        super().exit()

borgmatic/actions/browse/archive.py

@@ -0,0 +1,143 @@
+import argparse
+import collections
+import enum
+import json
+import logging
+
+import binaryornot.helpers
+
+import borgmatic.borg.extract
+import borgmatic.borg.list
+import borgmatic.borg.repo_list
+import borgmatic.borg.version
+
+logger = logging.getLogger(__name__)
+
+
+class Path_type(enum.Enum):
+    DIRECTORY = 'd'
+    LINK = 'l'
+    PIPE = 'p'
+    FILE = '-'
+
+
+# A data structure capturing a path stored in a Borg archive.
+Archive_path = collections.namedtuple(
+    'Archive_path',
+    ('path_type', 'file_path', 'link_target'),
+)
+
+
+def get_repository_archives(config, repository):
+    '''
+    Given a configuration dict and a repository dict, return a list of the repository's archives,
+    one dict per archive.
+    '''
+    with borgmatic.logger.Log_prefix(repository.get('label', repository['path'])):
+        logger.info('Listing repository')
+        repo_list_arguments = argparse.Namespace(
+            repository=repository['path'],
+            short=None,
+            format=None,
+            json=True,
+            prefix=None,
+            match_archives=None,
+            sort_by=None,
+            first=None,
+            last=None,
+        )
+        global_arguments = argparse.Namespace()
+        local_path = config.get('local_path', 'borg')
+        remote_path = config.get('remote_path')
+        local_borg_version = borgmatic.borg.version.local_borg_version(config, local_path)
+
+        return json.loads(
+            borgmatic.borg.repo_list.list_repository(
+                repository['path'],
+                config,
+                local_borg_version,
+                repo_list_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        )
+
+
+def get_archive_paths(config, repository, archive_name):
+    '''
+    Given a configuration dict, a repository dict, and an archive name in that repository, get a
+    generator of files, directories, symlinks, etc. found in the archive, each as an Archive_path
+    instance.
+    '''
+    with borgmatic.logger.Log_prefix(repository.get('label', repository['path'])):
+        logger.info(f'Listing archive {archive_name}')
+
+        global_arguments = argparse.Namespace()
+        local_path = config.get('local_path', 'borg')
+        remote_path = config.get('remote_path')
+        local_borg_version = borgmatic.borg.version.local_borg_version(config, local_path)
+
+        return (
+            Archive_path(
+                path_data['type'],
+                path_data['path'],
+                path_data.get('linktarget'),
+            )
+            for path_data in borgmatic.borg.list.capture_archive_listing(
+                repository['path'],
+                archive_name,
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+        )
+
+
+READLINES_HINT_BYTES = 100000
+TRUNCATION_MESSAGE = '[... truncated for display ...]'
+
+
+def get_archive_file_content(config, repository, archive_name, file_path):
+    '''
+    Given a configuration dict, a repository dict, an archive name in that repository, and a file
+    path in that archive, return the file's contents or None if the file can't be loaded, e.g.
+    because it's binary or can't be decoded.
+
+    If the file is too large, then truncate the returned content.
+    '''
+    with borgmatic.logger.Log_prefix(repository.get('label', repository['path'])):
+        logger.info(f'Getting archive content of file {file_path}')
+        local_path = config.get('local_path', 'borg')
+        remote_path = config.get('remote_path')
+
+        lines = borgmatic.borg.extract.extract_archive(
+            dry_run=False,
+            repository=repository['path'],
+            archive=archive_name,
+            paths=(file_path,),
+            config=config,
+            local_borg_version=borgmatic.borg.version.local_borg_version(config, local_path),
+            global_arguments=argparse.Namespace(),
+            local_path=local_path,
+            remote_path=remote_path,
+            destination_path=None,
+            strip_components=None,
+            extract_to_stdout=True,
+        ).stdout.readlines(READLINES_HINT_BYTES)
+
+        content = b''.join(lines)
+
+        if binaryornot.helpers.is_binary_string(content):
+            return None
+
+        try:
+            return (
+                content.decode()
+                if len(content) < READLINES_HINT_BYTES
+                else f'{content.decode()}\n{TRUNCATION_MESSAGE}'
+            )
+        except UnicodeDecodeError:
+            return None

borgmatic/actions/browse/archives_list.py

@@ -0,0 +1,89 @@
+import textual.widgets
+
+import borgmatic.actions.browse.bindings
+import borgmatic.actions.browse.loading
+import borgmatic.actions.browse.workers
+
+
+class Archives_list(textual.widgets.OptionList):
+    '''
+    A widget for selecting a single Borg archive from among the archives in a repository. The item
+    selection event is handled in a Carousel instance, the parent widget of an Archives_list.
+    '''
+
+    BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
+
+    def __init__(self, config, repository):
+        '''
+        Given a configuration dict and a repository dict, prepare to load the archives from the
+        repository for eventual display in this widget. Actual loading kicks off in on_mount()
+        below.
+        '''
+        self.config = config
+        self.repository = repository
+
+        super().__init__(classes='panel')
+        self.border_title = '📚 archives'
+        self.highlighted_option_changed = False
+        self.archive_loaded = borgmatic.actions.browse.workers.Archive_loaded(
+            self, 'archive loaded'
+        )
+
+        self.loading_timer = borgmatic.actions.browse.loading.add_inline_loading_indicator(self)
+
+    def on_mount(self):
+        '''
+        When this widget gets mounted in the DOM, subscribe to archive loaded events so that we can
+        find out about archives as they load. Also start loading archives from the repository.
+
+        Loading is started *after* subscribing to the archive loaded signal so that there's not a
+        gap where we might miss out on signal publishes.
+        '''
+        self.archive_loaded.subscribe(self, self.on_archive_loaded)
+
+        borgmatic.actions.browse.workers.add_repository_archives(
+            self.app,
+            archive_loaded=self.archive_loaded,
+            config=self.config,
+            repository=self.repository,
+        )
+
+    def on_archive_loaded(self, archive_name):
+        '''
+        When an archive loads, add it as an option to this archives list. But if we get a
+        signal that all path loading is complete, stop and remove our loading indicator.
+        '''
+        if archive_name is borgmatic.actions.browse.workers.LOADING_DONE:
+            self.loading_timer.stop()
+            self.remove_option('loading-indicator')
+            return
+
+        label_pieces = (
+            (archive_name, '[dim](latest)[/dim]') if len(self.options) == 1 else (archive_name,)
+        )
+        highlighted_option = self.highlighted_option
+
+        loading_indicator = self.get_option('loading-indicator')
+        self.remove_option('loading-indicator')
+        self.add_options(
+            (
+                textual.widgets.option_list.Option(' '.join(label_pieces), id=archive_name),
+                loading_indicator,
+            ),
+        )
+
+        # Retain the highlighted option position even as other options load around it.
+        self.highlighted = (
+            self.get_option_index(highlighted_option.id)
+            if highlighted_option and self.highlighted_option_changed
+            else 0
+        )
+
+    def on_option_list_option_highlighted(self, event):
+        '''
+        When the highlighted option changes, record that fact. This flag is consumed in
+        borgmatic.actions.browse.workers.add_repository_archives() in order to retain the
+        highlighted option even as other options load around it.
+        '''
+        if self.highlighted not in {None, 0}:
+            self.highlighted_option_changed = True

borgmatic/actions/browse/bindings.py

@@ -0,0 +1,22 @@
+import textual.binding
+import textual.widgets
+
+OPTION_LIST_BINDINGS = (
+    *textual.widgets.OptionList.BINDINGS,
+    textual.binding.Binding(
+        key='up,k', action='cursor_up', description='scroll up', show=True, priority=True
+    ),
+    textual.binding.Binding(
+        key='down,j', action='cursor_down', description='scroll down', show=True, priority=True
+    ),
+    textual.binding.Binding(
+        key='pageup', action='page_up', description='page up', show=True, priority=True
+    ),
+    textual.binding.Binding(
+        key='pagedown', action='page_down', description='page down', show=True, priority=True
+    ),
+    textual.binding.Binding(
+        key='enter', action='select', description='select', show=True, priority=True
+    ),
+    textual.binding.Binding(key='right,l', action='select', description='select', show=False),
+)

borgmatic/actions/browse/carousel.py

@@ -0,0 +1,153 @@
+import os
+
+import textual.binding
+import textual.containers
+
+import borgmatic.actions.browse.archive
+import borgmatic.actions.browse.archives_list
+import borgmatic.actions.browse.configuration_files_list
+import borgmatic.actions.browse.directory_list
+import borgmatic.actions.browse.file_preview
+import borgmatic.actions.browse.icons
+import borgmatic.actions.browse.repositories_list
+
+
+def make_next_panel(focused_panel, option_id):
+    '''
+    Given a focused panel widget and the selected option ID, return the next panel corresponding to
+    that selection. This is the mechanism by which the user can successively drill down from
+    configuration file to repository to archive to root directory to non-root directory or file.
+
+    If the particular option ID on the focused panel doesn't have a supported next panel, then
+    return None.
+    '''
+    if isinstance(
+        focused_panel, borgmatic.actions.browse.configuration_files_list.Configuration_files_list
+    ):
+        return borgmatic.actions.browse.repositories_list.Repositories_list(
+            config=focused_panel.configs[option_id]
+        )
+
+    if isinstance(focused_panel, borgmatic.actions.browse.repositories_list.Repositories_list):
+        return borgmatic.actions.browse.archives_list.Archives_list(
+            config=focused_panel.config, repository=focused_panel.repositories[option_id]
+        )
+
+    if isinstance(focused_panel, borgmatic.actions.browse.archives_list.Archives_list):
+        return borgmatic.actions.browse.directory_list.Directory_list(
+            config=focused_panel.config, repository=focused_panel.repository, archive_name=option_id
+        )
+
+    if isinstance(focused_panel, borgmatic.actions.browse.directory_list.Directory_list):
+        option = focused_panel.get_option(option_id)
+
+        if option.prompt.startswith(
+            borgmatic.actions.browse.icons.PATH_TYPE_ICONS[
+                borgmatic.actions.browse.archive.Path_type.DIRECTORY.value
+            ]
+        ):
+            return borgmatic.actions.browse.directory_list.Directory_list(
+                focused_panel.config,
+                focused_panel.repository,
+                focused_panel.archive_name,
+                path_loaded=focused_panel.path_loaded,
+                path_components=(*focused_panel.path_components, option_id),
+            )
+
+        if option.prompt.startswith(
+            borgmatic.actions.browse.icons.PATH_TYPE_ICONS[
+                borgmatic.actions.browse.archive.Path_type.FILE.value
+            ]
+        ):
+            return borgmatic.actions.browse.file_preview.File_preview(
+                focused_panel.config,
+                focused_panel.repository,
+                focused_panel.archive_name,
+                file_path=os.path.sep.join((*focused_panel.path_components, option_id)),
+            )
+
+    return None
+
+
+class Carousel(textual.containers.Horizontal):
+    BINDINGS = (
+        textual.binding.Binding(
+            key='left,h', action='previous', description='previous', priority=True
+        ),
+    )
+
+    def __init__(self, panels):
+        self.panels = panels
+        self.focused_panel = panels[0]
+
+        super().__init__()
+
+    def compose(self):
+        '''
+        Compose with each of the contained panels and focus the first one.
+        '''
+        yield from self.panels
+
+        self.focused_panel.focus()
+
+    def action_previous(self):
+        '''
+        Make the previous panel into the focused panel.
+        '''
+        previous_panel_index = self.panels.index(self.focused_panel) - 1
+
+        if previous_panel_index < 0:
+            return
+
+        self.focused_panel.styles.display = 'none'
+
+        self.focused_panel = self.panels[previous_panel_index]
+        self.focused_panel.styles.display = 'block'
+        self.focused_panel.focus()
+
+    def action_next(self, option_id):
+        '''
+        Hide the current focused panel and create the next one.
+        '''
+        next_panel_index = self.panels.index(self.focused_panel) + 1
+
+        if next_panel_index < len(self.panels):
+            next_panel = self.panels[next_panel_index]
+            next_panel.styles.display = 'block'
+        else:
+            next_panel = make_next_panel(self.focused_panel, option_id)
+
+            if next_panel is None:
+                self.notify('Cannot display this content', severity='warning')
+                return
+
+            self.panels.append(next_panel)
+            next_panel.highlighted = 0
+
+        self.focused_panel.styles.display = 'none'
+        self.focused_panel = next_panel
+        self.focused_panel.focus()
+        self.mount(self.focused_panel)
+
+    def on_option_list_option_highlighted(self, event):
+        '''
+        The highlighted option has changed, so truncate any next panels.
+        '''
+        next_panel_index = self.panels.index(self.focused_panel) + 1
+
+        del self.panels[next_panel_index:]
+
+    def on_option_list_option_selected(self, event):
+        '''
+        An option has been selected, so advance to the next panel—unless the option selected is
+        "..", in which case go to the previous panel.
+        '''
+        if (
+            event.option_list != self.focused_panel or event.option_id == 'loading-indicator'
+        ):  # pragma: no cover
+            return
+
+        if event.option_id == '..':
+            self.action_previous()
+        else:
+            self.action_next(event.option_id)

borgmatic/actions/browse/configuration_files_list.py

@@ -0,0 +1,33 @@
+import os
+
+import textual.widgets
+
+import borgmatic.actions.browse.bindings
+
+
+class Configuration_files_list(textual.widgets.OptionList):
+    '''
+    A widget for selecting a single borgmatic configuration file from among available configuration
+    files. The item selection event is handled in a Carousel instance, the parent widget of an
+    Configuration_files_list.
+    '''
+
+    BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
+
+    def __init__(self, configs):
+        '''
+        Given a dict mapping from configuration path to corresponding configuration dict, add each
+        configuration path as an option to this widget.
+        '''
+        self.configs = configs
+        home_directory = os.path.expanduser('~')
+
+        super().__init__(
+            *(
+                textual.widgets.option_list.Option(unexpanded_path, id=config_path)
+                for config_path in configs
+                for unexpanded_path in (config_path.replace(home_directory, '~'),)
+            ),
+            classes='panel',
+        )
+        self.border_title = '📄 configuration files'

borgmatic/actions/browse/directory_list.py

@@ -0,0 +1,211 @@
+import os
+
+import textual.widgets
+
+import borgmatic.actions.browse.bindings
+import borgmatic.actions.browse.icons
+import borgmatic.actions.browse.loading
+import borgmatic.actions.browse.workers
+
+
+def get_relative_archive_path_components(archive_path, current_directory_path_components):
+    '''
+    Given an Archive_path instance and a tuple of path components for the currently browsed
+    directory, get the path components as a tuple for the archive path relative to that directory.
+
+    For instance, given an archive path with a path of 'foo/bar/baz/quux.txt' and current
+    directory path components of ('foo', 'bar'), return ('baz', 'quux.txt').
+
+    If the archive path is not actually relative to the current directory, return None.
+    '''
+    archive_path_components = tuple(archive_path.file_path.split(os.path.sep))
+
+    if not current_directory_path_components:
+        return archive_path_components
+
+    # If the loaded path doesn't match this directory list's own path, then we don't care about
+    # it for purposes of displaying this particular directory.
+    if (
+        tuple(archive_path_components[: len(current_directory_path_components)])
+        != current_directory_path_components
+    ):
+        return None
+
+    # Strip off the portion of the archive path that matches the directory list's own path.
+    return archive_path_components[len(current_directory_path_components) :]
+
+
+def make_directory_list_option(archive_path, relative_path_components):
+    '''
+    Given an Archive_path instance and a tuple of relative path components for it, make a
+    textual.widgets.option_list.Option for the path. Use an the icon based on whether this looks
+    like a terminal filename or a directory.
+    '''
+    pieces = (
+        borgmatic.actions.browse.icons.PATH_TYPE_ICONS.get(
+            archive_path.path_type if len(relative_path_components) == 1 else 'd', '❓'
+        ),
+        relative_path_components[0],
+    ) + (('→', archive_path.link_target) if archive_path.link_target else ())
+
+    return textual.widgets.option_list.Option(
+        prompt=' '.join(pieces), id=relative_path_components[0]
+    )
+
+
+def add_archive_paths(
+    directory_list,
+    config,
+    repository,
+    archive_name,
+    archive_paths,
+):
+    '''
+    Given a DirectoryList instance, a configuration dict, a repository dict, an archive name, and a
+    sequence of ArchivePath instances, add the paths to the directory list as options, sorting and
+    deduplicating the resulting directory list's options.
+
+    After all of this reshuffling, make sure the original highlighted option remains highlighted.
+    '''
+    highlighted_option = directory_list.highlighted_option
+    original_options_count = len(directory_list.options)
+
+    sorted_options = sorted(
+        (
+            *directory_list.options,
+            *(
+                make_directory_list_option(archive_path, relative_path_components)
+                for archive_path in archive_paths
+                for relative_path_components in (
+                    get_relative_archive_path_components(
+                        archive_path,
+                        directory_list.path_components,
+                    ),
+                )
+                if relative_path_components
+                if relative_path_components[0] not in directory_list._id_to_option
+            ),
+        ),
+        # The loading indicator "option" always goes to the bottom.
+        key=lambda option: ((option.id == 'loading-indicator'), option.prompt),
+    )
+
+    # If there aren't actually any options to add (due to deduplication), bail.
+    if len(sorted_options) == original_options_count:
+        return
+
+    # Retain the highlighted option position even as other options load around it.
+    directory_list.set_options(sorted_options)
+    directory_list.highlighted = (
+        directory_list.get_option_index(highlighted_option.id)
+        if highlighted_option and directory_list.highlighted_option_changed
+        else 0
+    )
+
+
+class Directory_list(textual.widgets.OptionList):
+    '''
+    A widget for selecting a path from among the contents of a particular directory in a Borg
+    archive. The item selection event is handled in a Carousel instance, the parent widget of a
+    Directory_list.
+    '''
+
+    BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
+
+    def __init__(self, config, repository, archive_name, path_loaded=None, path_components=None):
+        '''
+        Given a configuration dict, a repository dict, an archive name, an optional
+        Archive_path_loaded instance for signalling new paths as they load, and an optional tuple of
+        path components indicating this directory's position in the backed up filesystem, prepare to
+        load paths from the archive for eventual display in this widget. Actual loading kicks off in
+        on_mount() below.
+        '''
+        self.config = config
+        self.repository = repository
+        self.archive_name = archive_name
+        self.path_components = path_components or ()
+        self.highlighted_option_changed = False
+
+        super().__init__(classes='panel')
+
+        self.border_title = ' '.join(
+            (
+                '📁',
+                os.path.sep.join(self.path_components)
+                if self.path_components
+                else f'{archive_name}',
+            )
+        )
+
+        if self.path_components:
+            self.add_option(
+                textual.widgets.option_list.Option(
+                    '📁 ..',
+                    id='..',
+                ),
+            )
+
+        self.path_loaded = path_loaded or borgmatic.actions.browse.workers.Archive_path_loaded(
+            self, 'archive path loaded'
+        )
+
+        if not self.path_loaded.complete:
+            self.timer = borgmatic.actions.browse.loading.add_inline_loading_indicator(self)
+
+    def on_mount(self):
+        '''
+        When this widget gets mounted in the DOM, subscribe to path loaded events so that we can
+        find out about relevant archive paths as they load. And if this is a root directory list,
+        start loading paths from the archive. If this is a non-root directory list, add any already
+        loaded archive paths to this widget as options.
+
+        Loading is started *after* subscribing to path loaded signals so that there's not a gap
+        where we might miss out on any paths.
+        '''
+        self.path_loaded.subscribe(self, self.on_archive_path_loaded)
+
+        if self.path_components:
+            add_archive_paths(
+                directory_list=self,
+                config=self.config,
+                repository=self.repository,
+                archive_name=self.archive_name,
+                archive_paths=borgmatic.actions.browse.workers.get_paths(
+                    self.path_loaded.path_hierarchy, self.path_components
+                ),
+            )
+        else:
+            borgmatic.actions.browse.workers.load_archive_paths(
+                self.app,
+                path_loaded=self.path_loaded,
+                config=self.config,
+                repository=self.repository,
+                archive_name=self.archive_name,
+            )
+
+    def on_archive_path_loaded(self, data):
+        '''
+        When an archive path loads, add it as an option to this directory list. But if we get a
+        signal that all path loading is complete, stop and remove our loading indicator.
+        '''
+        if data is borgmatic.actions.browse.workers.LOADING_DONE:
+            self.timer.stop()
+            self.remove_option('loading-indicator')
+            return
+
+        add_archive_paths(
+            directory_list=self,
+            config=self.config,
+            repository=self.repository,
+            archive_name=self.archive_name,
+            archive_paths=(data,),
+        )
+
+    def on_option_list_option_highlighted(self, event):
+        '''
+        When the highlighted option changes, record that fact. This flag is consumed in
+        add_archive_paths() in order to retain the highlighted option even as other options load
+        around it.
+        '''
+        if self.highlighted not in {None, 0}:
+            self.highlighted_option_changed = True

borgmatic/actions/browse/file_preview.py

@@ -0,0 +1,87 @@
+import logging
+
+import rich.syntax
+import textual.binding
+import textual.widgets
+
+import borgmatic.actions.browse.loading
+import borgmatic.actions.browse.workers
+
+logger = logging.getLogger('__name__')
+
+
+class File_preview(textual.widgets.RichLog):
+    '''
+    A widget for extracting and previewing the contents of a file stored in a Borg archive.
+    '''
+
+    BINDINGS = (
+        *textual.widgets.RichLog.BINDINGS,
+        textual.binding.Binding(
+            key='up,k', action='scroll_up', description='scroll up', show=True, priority=True
+        ),
+        textual.binding.Binding(
+            key='down,j', action='scroll_down', description='scroll down', show=True, priority=True
+        ),
+        textual.binding.Binding(
+            key='pageup', action='page_up', description='page up', show=True, priority=True
+        ),
+        textual.binding.Binding(
+            key='pagedown', action='page_down', description='page down', show=True, priority=True
+        ),
+    )
+
+    def __init__(self, config, repository, archive_name, file_path):
+        '''
+        Given a configuration dict, a repository dict, an archive name, and the path of a file in
+        the archive, prepare to load the file's contents for eventual display in this widget. Actual
+        loading kicks off in on_mount() below.
+        '''
+        self.config = config
+        self.repository = repository
+        self.archive_name = archive_name
+        self.file_path = file_path
+
+        super().__init__(classes='panel')
+        self.border_title = f'📄 {self.file_path} preview'
+        self.auto_scroll = False
+        self.file_preview_loaded = borgmatic.actions.browse.workers.File_preview_loaded(
+            self, 'file preview loaded'
+        )
+
+        self.loading_timer = borgmatic.actions.browse.loading.add_inline_loading_indicator(self)
+
+    def on_mount(self):
+        '''
+        When this widget gets mounted in the DOM, subscribe to archive loaded events so that we can
+        find out about archives as they load. Also start loading file contents from the archive.
+
+        Loading is started *after* subscribing to the file preview loaded signal so that there's not
+        a gap where we might miss out on signal publishes.
+        '''
+        self.file_preview_loaded.subscribe(self, self.on_file_preview_loaded)
+
+        borgmatic.actions.browse.workers.load_file_preview(
+            self.app,
+            file_preview_loaded=self.file_preview_loaded,
+            config=self.config,
+            repository=self.repository,
+            archive_name=self.archive_name,
+            file_path=self.file_path,
+        )
+
+    def on_file_preview_loaded(self, file_contents):
+        '''
+        When a file loads, write its contents (syntax highlighted) to this file preview widget.
+        '''
+        self.loading_timer.stop()
+        self.clear()
+
+        if file_contents is None:
+            self.write('Cannot display a preview for this file')
+        else:
+            # Only pass the file path and not its contents to guess_lexer(). Passing the contents is
+            # more accurate, but also much slower.
+            self.write(
+                rich.syntax.Syntax(file_contents, rich.syntax.Syntax.guess_lexer(self.file_path))
+            )

borgmatic/actions/browse/icons.py

@@ -0,0 +1,8 @@
+import borgmatic.actions.browse.archive
+
+PATH_TYPE_ICONS = {
+    borgmatic.actions.browse.archive.Path_type.DIRECTORY.value: '📁',
+    borgmatic.actions.browse.archive.Path_type.LINK.value: '🔗',
+    borgmatic.actions.browse.archive.Path_type.PIPE.value: '🚰',
+    borgmatic.actions.browse.archive.Path_type.FILE.value: '📄',
+}

borgmatic/actions/browse/loading.py

@@ -0,0 +1,54 @@
+import contextlib
+import functools
+import logging
+
+import textual.widgets
+import textual.widgets.option_list
+
+LOADING_DOT_INTERVAL_SECONDS = 0.3
+
+
+logger = logging.getLogger('__name__')
+
+
+def update_inline_loading_indicator(widget):
+    '''
+    Given a textual.widgets.OptionList or a textual.widgets.RichLog instance, animate the existing
+    loading indicator inside it.
+    '''
+    if isinstance(widget, textual.widgets.OptionList):
+        with contextlib.suppress(textual.widgets.option_list.OptionDoesNotExist):
+            widget.replace_option_prompt(
+                'loading-indicator',
+                (str(widget.get_option('loading-indicator').prompt) + '.').replace('....', ''),
+            )
+    elif isinstance(widget, textual.widgets.RichLog):
+        with contextlib.suppress(IndexError):
+            loading_message = str(widget.lines[0].text)
+            widget.clear()
+            widget.write((loading_message + '.').replace('....', ''))
+    else:
+        raise ValueError(f'Unsupported widget type: {type(widget)}')
+
+
+LOADING_MESSAGE = '⏳ loading...'
+
+
+def add_inline_loading_indicator(widget):
+    '''
+    Given a textual.widgets.OptionList or a textual.widgets.RichLog instance, add a loading
+    indicator to it.
+    '''
+    if isinstance(widget, textual.widgets.OptionList):
+        loading_option = textual.widgets.option_list.Option(LOADING_MESSAGE, id='loading-indicator')
+        widget.add_option(loading_option)
+        widget.highlighted = None
+    elif isinstance(widget, textual.widgets.RichLog):
+        widget.write(LOADING_MESSAGE)
+    else:
+        raise ValueError(f'Unsupported widget type: {type(widget)}')
+
+    return widget.set_interval(
+        LOADING_DOT_INTERVAL_SECONDS,
+        functools.partial(update_inline_loading_indicator, widget),
+    )

borgmatic/actions/browse/logs.py

@@ -0,0 +1,102 @@
+import contextlib
+import logging
+
+import textual._context
+import textual.widgets
+import textual.worker
+
+import borgmatic.logger
+
+
+class Rich_color_formatter(logging.Formatter):
+    '''
+    A Python logging formatter that formats log records with Rich-compatible color markup according
+    to their levels.
+    '''
+
+    def __init__(self, *args, **kwargs):
+        self.prefix = None
+        super().__init__(
+            '{prefix}{message}',
+            *args,
+            style='{',
+            **kwargs,
+        )
+
+    def format(self, record):
+        '''
+        Given a log record, format it with Rich-compatibe color markup corresponding to its log
+        level.
+        '''
+        borgmatic.logger.add_custom_log_levels()
+
+        color = {
+            logging.CRITICAL: 'bright_red',
+            logging.ERROR: 'bright_red',
+            logging.WARNING: 'bright_yellow',
+            logging.ANSWER: 'bright_magenta',
+            logging.INFO: 'bright_green',
+            logging.DEBUG: 'bright_cyan',
+        }.get(record.levelno)
+        record.prefix = f'{self.prefix}: ' if self.prefix else ''
+
+        return f'[{color}]{super().format(record)}[/{color}]'
+
+
+class Browse_log_handler(logging.Handler):
+    '''
+    A Python log handler that writes any log records to a logging widget.
+    '''
+
+    def __init__(self, logs_widget):
+        '''
+        Given a logs widget, save it for use below.
+        '''
+        self.logs_widget = logs_widget
+
+        super().__init__()
+
+    def emit(self, record):
+        '''
+        Given a log record, format it and log it to the logs widgets. This works whether or not the
+        logging is happening in the main thread.
+        '''
+        message = self.format(record)
+
+        try:
+            textual.worker.get_current_worker()
+            self.logs_widget.app.call_from_thread(self.logs_widget.write, message)
+        except (RuntimeError, textual.worker.NoActiveWorker):
+            with contextlib.suppress(textual._context.NoActiveAppError):
+                self.logs_widget.write(message)
+
+
+def log_to_widget(logs_widget):
+    '''
+    Given a Textual RichLog logs widget, add a log handler and formatter that logs to it. Also
+    remove the default borgmatic console log handler so it doesn't try to log all over our UI.
+    '''
+    handler = Browse_log_handler(logs_widget)
+    handler.setFormatter(Rich_color_formatter())
+    logger = logging.getLogger()
+    logger.setLevel(min(handler.level for handler in logger.handlers))
+    logger.addHandler(handler)
+
+    with contextlib.suppress(StopIteration):
+        console_handler = next(
+            handler
+            for handler in logging.getLogger().handlers
+            if isinstance(handler, borgmatic.logger.Multi_stream_handler)
+        )
+        logger.removeHandler(console_handler)
+
+
+class Logs(textual.widgets.RichLog):
+    '''
+    A widget for viewing borgmatic logs in realtime. The log level is determined by borgmatic's
+    current verbosity level.
+    '''
+
+    def __init__(self):
+        super().__init__(markup=True, id='logs', classes='panel')
+        self.border_title = '🪵 logs'

borgmatic/actions/browse/repositories_list.py

@@ -0,0 +1,30 @@
+import textual.widgets
+
+import borgmatic.actions.browse.bindings
+
+
+class Repositories_list(textual.widgets.OptionList):
+    '''
+    A widget for selecting a single Borg repository from among the repositories in a borgmatic
+    configuration file. The item selection event is handled in a Carousel instance, the parent
+    widget of a Repositories_list.
+    '''
+
+    BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
+
+    def __init__(self, config):
+        '''
+        Given a configuration dict, populate the repositories in this widget.
+        '''
+        self.config = config
+        self.repositories = config['repositories']
+
+        super().__init__(
+            *(
+                textual.widgets.option_list.Option(label, id=index)
+                for index, repository in enumerate(self.repositories)
+                for label in (repository.get('label', repository.get('path')),)
+            ),
+            classes='panel',
+        )
+        self.border_title = '📦 repositories'

borgmatic/actions/browse/run.py

@@ -0,0 +1,29 @@
+import logging
+
+
+def run_browse(
+    diff_arguments,
+    global_arguments,
+    configs,
+):
+    '''
+    Run the "browse" action for the given borgmatic configurations. This launches a console UI.
+
+    Raise ValueError if the Textual library (a prerequisite for this action) can't be imported.
+    '''
+    if not configs:
+        return
+
+    logging.getLogger('asyncio').setLevel(logging.WARNING)
+
+    try:
+        import textual  # noqa: F401, PLC0415
+    except ImportError:  # pragma: no cover
+        raise ValueError(
+            'Unable to import the Textual library for the browse action; try installing "borgmatic[browse]"'
+        )
+
+    import borgmatic.actions.browse.app  # noqa: PLC0415
+
+    app = borgmatic.actions.browse.app.Browse_app(configs)
+    app.run()

borgmatic/actions/browse/workers.py

@@ -0,0 +1,192 @@
+import logging
+import os
+
+import textual
+import textual.signal
+
+import borgmatic.actions.browse.archive
+
+logger = logging.getLogger('__name__')
+
+
+LOADING_DONE = object()
+
+
+class Archive_loaded(textual.signal.Signal):
+    '''
+    A signal that publishes when each subsequent archive is loaded from a repository, intended for
+    consumption in widgets that display archives as they are loaded. This signal also publishes
+    when loading is complete.
+
+    Each subscribed callback call includes the archive as an archive name string. Given the lack of
+    other identifying information (configuration file, repository), there should be a separate
+    Archive_loaded instance per repository.
+    '''
+
+
+@textual.work(thread=True)
+def add_repository_archives(browse_app, archive_loaded, config, repository):
+    '''
+    Given a running Browse_app instance, an Archive_loaded instance, a configuration dict, and a
+    repository dict, load a list of the archives from the repository and add them as options in the
+    archives list. Reverse the order so the most recent archive is first.
+
+    This function runs in a separate thread from the main UI. When loading is complete, publish a
+    loading done signal.
+    '''
+    archives_data = borgmatic.actions.browse.archive.get_repository_archives(config, repository)
+
+    # Reverse the archives, so the common case of accessing the latest archive is easy because it's
+    # at the top.
+    for archive in reversed(archives_data['archives']):
+        archive_loaded.publish(archive['archive'])
+
+    archive_loaded.publish(LOADING_DONE)
+
+
+def record_path(archive_path, hierarchy, path_components):
+    '''
+    Given an Archive_path instance, a dict capturing a filesystem hierarchy of paths, and a tuple of
+    path components for the archive path, set the archive path into the hierarchy data structure.
+
+    For instance, if given an archive path and path components representing "foo/bar/baz.txt",
+    produce a hierarchy that looks like:
+
+        {'foo': {'bar': {'baz.txt': Archive_path('-', 'foo/bar/baz.txt', '')}}}
+
+    Note that the hierarchy is modified in place, so any existing paths there are retained.
+    '''
+    if len(path_components) == 1:
+        hierarchy[path_components[0]] = {} if archive_path.path_type == 'd' else archive_path
+        return
+
+    record_path(archive_path, hierarchy.setdefault(path_components[0], {}), path_components[1:])
+
+
+def get_paths(hierarchy, path_components, full_path_components=None):
+    '''
+    Given a dict capturing a filesystem hierarchy of paths (or a subset thereof), a tuple of path
+    components for a directory path relative to the hierarchy root, and an optional tuple of
+    *absolute* path components for the same path (if different), return a generator of the
+    contained file and directory Archive_path instances from the hierarchy.
+
+    For instance, given the following hierarchy:
+
+        {'foo': {'bar': {'baz.txt': Archive_path('-', 'foo/bar/baz.txt', ''), 'quux': {}}}}
+
+    ... and path components of ('foo', 'bar'), return a generator with the following:
+
+        * Archive_path('-', 'foo/bar/baz.txt', '')
+        * Archive_path('d', 'foo/bar/quux', '')
+
+    The given absolute path components are use to construct directory paths like that last archive
+    path.
+    '''
+    if full_path_components is None:
+        full_path_components = path_components
+
+    if len(path_components) == 1:
+        try:
+            return (
+                archive_path
+                if isinstance(archive_path, borgmatic.actions.browse.archive.Archive_path)
+                else borgmatic.actions.browse.archive.Archive_path(
+                    'd', os.path.join(*full_path_components, component), ''
+                )
+                for component, archive_path in hierarchy[path_components[0]].items()
+            )
+        except KeyError:
+            raise ValueError(f'Unknown file or directory: {path_components[0]}')
+
+    try:
+        return get_paths(hierarchy[path_components[0]], path_components[1:], full_path_components)
+    except KeyError:
+        raise ValueError(f'Unknown directory: {path_components[0]}')
+
+
+class Archive_path_loaded(textual.signal.Signal):
+    '''
+    A signal that publishes when each subsequent path is loaded from an archive, intended for
+    consumption in widgets that display paths as they are loaded. This signal also tracks the
+    complete filesystem hierarchy seen thus far, so new widgets that get created after loading has
+    started can "catch up" with existing known paths. Lastly, this signal publishes and tracks when
+    loading is complete.
+
+    Each subscrided callback call includes the loaded path as an Archive_path instance. There is
+    intended to be a separate Archive_path_loaded instance per archive, but that instance should be
+    shared among several different widgets for the same archive for performance reasons.
+    '''
+
+    def __init__(self, owner, name):
+        self.path_hierarchy = {}
+        self.complete = False
+
+        super().__init__(owner, name)
+
+    def publish(self, archive_path):
+        '''
+        Publish the given archive path to subscribers and record its path locally. But if the
+        archive path is actually LOADING_DONE, then record loading as complete.
+        '''
+        super().publish(archive_path)
+
+        if archive_path is LOADING_DONE:
+            self.complete = True
+        else:
+            record_path(
+                archive_path, self.path_hierarchy, archive_path.file_path.split(os.path.sep)
+            )
+
+
+@textual.work(thread=True)
+def load_archive_paths(browse_app, path_loaded, config, repository, archive_name):
+    '''
+    Given a running Browse_app instance, an Archive_path_loaded instance, a configuration dict, a
+    repository dict, and an archive name, load the paths in this archive and publish each one via
+    the Archive_path_loaded signal, so interested widgets can subscribe. Also send a "loading done"
+    signal when loading completes.
+
+    This function runs in a separate thread from the main UI. When loading is complete, publish a
+    loading done signal.
+    '''
+    for archive_path in borgmatic.actions.browse.archive.get_archive_paths(
+        config, repository, archive_name
+    ):
+        path_loaded.publish(archive_path)
+
+    path_loaded.publish(LOADING_DONE)
+
+
+class File_preview_loaded(textual.signal.Signal):
+    '''
+    A signal that publishes when file contents are loaded from an archive, intended for consumption
+    in widgets that display loaded files. This signal also publishes when loading is complete.
+
+    Each published callback includes a the file's contents as a string. Given the lack of other
+    identifying information (configuration file, repository, archive), there should be a separate
+    Archive_loaded instance per previewed file.
+    '''
+
+
+@textual.work(thread=True)
+def load_file_preview(
+    browse_app,
+    file_preview_loaded,
+    config,
+    repository,
+    archive_name,
+    file_path,
+):
+    '''
+    Given a running Browse_app instance, a File_preview_loaded instance, a configuration dict, a
+    repository dict, an archive name, and the path of a file in that archive, load the contents of
+    the file and write it into the given file preview widget.
+
+    This function runs in a separate thread from the main UI. When loading is complete, publish a
+    loading done signal.
+    '''
+    file_contents = borgmatic.actions.browse.archive.get_archive_file_content(
+        config, repository, archive_name, file_path
+    )
+
+    file_preview_loaded.publish(file_contents)

borgmatic/actions/change_passphrase.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.change_passphrase
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -16,23 +15,15 @@ def run_change_passphrase(
     remote_path,
 ):
     '''
-    Run the "key change-passprhase" action for the given repository.
+    Run the "key change-passphrase" action for the given repository.
     '''
-    if (
+    logger.info('Changing repository passphrase')
-        change_passphrase_arguments.repository is None
+    borgmatic.borg.change_passphrase.change_passphrase(
-        or borgmatic.config.validate.repositories_match(
+        repository['path'],
-            repository, change_passphrase_arguments.repository
+        config,
-        )
+        local_borg_version,
-    ):
+        change_passphrase_arguments,
-        logger.info(
+        global_arguments,
-            f'{repository.get("label", repository["path"])}: Changing repository passphrase'
+        local_path=local_path,
-        )
+        remote_path=remote_path,
-        borgmatic.borg.change_passphrase.change_passphrase(
+    )
-            repository['path'],
-            config,
-            local_borg_version,
-            change_passphrase_arguments,
-            global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/check.py

@@ -1,4 +1,5 @@
 import calendar
+import contextlib
 import datetime
 import hashlib
 import itertools
@@ -6,20 +7,22 @@ import logging
 import os
 import pathlib
 import random
+import shlex
 import shutil
+import subprocess
+import textwrap
 
-import borgmatic.actions.create
+import borgmatic.actions.config.bootstrap
+import borgmatic.actions.pattern
 import borgmatic.borg.check
 import borgmatic.borg.create
 import borgmatic.borg.environment
 import borgmatic.borg.extract
 import borgmatic.borg.list
+import borgmatic.borg.pattern
 import borgmatic.borg.repo_list
-import borgmatic.borg.state
 import borgmatic.config.paths
-import borgmatic.config.validate
 import borgmatic.execute
-import borgmatic.hooks.command
 
 DEFAULT_CHECKS = (
     {'name': 'repository', 'frequency': '1 month'},
@@ -53,12 +56,14 @@ def parse_checks(config, only_checks=None):
 
     if 'disabled' in checks:
         logger.warning(
-            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
+            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead',
         )
+
         if len(checks) > 1:
             logger.warning(
-                'Multiple checks are configured, but one of them is "disabled"; not running any checks'
+                'Multiple checks are configured, but one of them is "disabled"; not running any checks',
             )
+
         return ()
 
     return checks
@@ -162,6 +167,7 @@ def filter_checks_on_frequency(
                     **dict.fromkeys(day for day in days if day != 'weekday'),
                     **dict.fromkeys(WEEKDAY_DAYS),
                 }
+
             if 'weekend' in days:
                 days = {
                     **dict.fromkeys(day for day in days if day != 'weekend'),
@@ -170,7 +176,7 @@ def filter_checks_on_frequency(
 
             if calendar.day_name[datetime_now().weekday()] not in days:
                 logger.info(
-                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)"
+                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)",
                 )
                 filtered_checks.remove(check)
                 continue
@@ -188,7 +194,7 @@ def filter_checks_on_frequency(
         if datetime_now() < check_time + frequency_delta:
             remaining = check_time + frequency_delta - datetime_now()
             logger.info(
-                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
+                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)',
             )
             filtered_checks.remove(check)
 
@@ -214,7 +220,7 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
     '''
     borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
 
-    if check_type in ('archives', 'data'):
+    if check_type in {'archives', 'data'}:
         return os.path.join(
             borgmatic_state_directory,
             'checks',
@@ -238,7 +244,7 @@ def write_check_time(path):  # pragma: no cover
     logger.debug(f'Writing check time at {path}')
 
     os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-    pathlib.Path(path, mode=0o600).touch()
+    pathlib.Path(path).touch(mode=0o600)
 
 
 def read_check_time(path):
@@ -249,7 +255,7 @@ def read_check_time(path):
     logger.debug(f'Reading check time from {path}')
 
     try:
-        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
+        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)  # noqa: DTZ006
     except FileNotFoundError:
         return None
 
@@ -280,7 +286,7 @@ def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
             (
                 make_check_time_path(config, borg_repository_id, check, archives_check_id),
                 make_check_time_path(config, borg_repository_id, check),
-            )
+            ),
         )
     )
 
@@ -312,16 +318,17 @@ def upgrade_check_times(config, borg_repository_id):
       {borgmatic_state_directory}/checks/1234567890/archives/all
     '''
     borgmatic_source_checks_path = os.path.join(
-        borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
+        borgmatic.config.paths.get_borgmatic_source_directory(config),
+        'checks',
     )
     borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
     borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
 
     if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
-        borgmatic_state_checks_path
+        borgmatic_state_checks_path,
     ):
         logger.debug(
-            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
+            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}',
         )
         os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
         shutil.move(borgmatic_source_checks_path, borgmatic_state_checks_path)
@@ -336,10 +343,8 @@ def upgrade_check_times(config, borg_repository_id):
 
         logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
 
-        try:
+        with contextlib.suppress(FileNotFoundError):
             shutil.move(old_path, temporary_path)
-        except FileNotFoundError:
-            pass
 
         os.mkdir(old_path)
         shutil.move(temporary_path, new_path)
@@ -353,60 +358,74 @@ def collect_spot_check_source_paths(
     local_path,
     remote_path,
     borgmatic_runtime_directory,
+    bootstrap_config_paths,
 ):
     '''
     Given a repository configuration dict, a configuration dict, the local Borg version, global
-    arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
+    arguments as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the
-    collect the source paths that Borg would use in an actual create (but only include files).
+    bootstrap configuration paths as read from an archive's manifest, collect the source paths that
+    Borg would use in an actual create (but only include files). As part of this, include the
+    bootstrap configuration paths, so that any configuration files included in the archive to
+    support bootstrapping are also spot checked.
     '''
     stream_processes = any(
         borgmatic.hooks.dispatch.call_hooks(
             'use_streaming',
             config,
-            repository['path'],
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-        ).values()
+        ).values(),
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, pattern_file) = (
+    (create_flags, create_positional_arguments, _) = borgmatic.borg.create.make_base_create_command(
-        borgmatic.borg.create.make_base_create_command(
+        dry_run=True,
-            dry_run=True,
+        repository_path=repository['path'],
-            repository_path=repository['path'],
+        # Omit "progress" because it interferes with "list_details".
-            config=config,
+        config=dict(config, progress=False, list_details=True),
-            patterns=borgmatic.actions.create.process_patterns(
+        patterns=borgmatic.actions.pattern.process_patterns(
-                borgmatic.actions.create.collect_patterns(config),
+            borgmatic.actions.pattern.collect_patterns(config, working_directory)
-                working_directory,
+            + tuple(
+                borgmatic.borg.pattern.Pattern(
+                    config_path,
+                    source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
+                )
+                for config_path in bootstrap_config_paths
             ),
-            local_borg_version=local_borg_version,
+            config,
-            global_arguments=global_arguments,
+            working_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
+        ),
-            local_path=local_path,
+        local_borg_version=local_borg_version,
-            remote_path=remote_path,
+        global_arguments=global_arguments,
-            list_files=True,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
-            stream_processes=stream_processes,
+        local_path=local_path,
-        )
+        remote_path=remote_path,
+        stream_processes=stream_processes,
     )
-    borg_environment = borgmatic.borg.environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    paths_output = borgmatic.execute.execute_command_and_capture_output(
+    path_lines = borgmatic.execute.execute_command_and_capture_output(
         create_flags + create_positional_arguments,
         capture_stderr=True,
-        extra_environment=borg_environment,
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
 
-    paths = tuple(
+    paths = (
         path_line.split(' ', 1)[1]
-        for path_line in paths_output.splitlines()
+        for path_line in path_lines
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
+        if path_line and path_line.startswith(('- ', '+ '))
     )
 
     return tuple(
-        path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
+        # Use dict.fromkeys() to deduplicate file paths, which are present in Borg's dry run output
+        # when there are overlapping source patterns. For instance, if both "/foo" and
+        # "/foo/file.txt" are in configured patterns, then "/foo/file.txt" will show up in Borg's
+        # dry run output twice.
+        dict.fromkeys(
+            path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
+        )
     )
 
 
@@ -436,28 +455,26 @@ def collect_spot_check_archive_paths(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return tuple(
-        path
+        entry['path']
-        for line in borgmatic.borg.list.capture_archive_listing(
+        for entry in borgmatic.borg.list.capture_archive_listing(
             repository['path'],
             archive,
             config,
             local_borg_version,
             global_arguments,
-            path_format='{type} {path}{NL}',  # noqa: FS003
             local_path=local_path,
             remote_path=remote_path,
         )
-        for (file_type, path) in (line.split(' ', 1),)
+        if entry['type'] not in {BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE}
-        if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
+        if pathlib.Path('borgmatic') not in pathlib.Path(entry['path']).parents
-        if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
         if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
-        not in pathlib.Path(path).parents
+        not in pathlib.Path(entry['path']).parents
         if pathlib.Path(borgmatic_runtime_directory.lstrip(os.path.sep))
-        not in pathlib.Path(path).parents
+        not in pathlib.Path(entry['path']).parents
     )
 
 
-SAMPLE_PATHS_SUBSET_COUNT = 10000
+SAMPLE_PATHS_SUBSET_COUNT = 5000
 
 
 def compare_spot_check_hashes(
@@ -468,31 +485,33 @@ def compare_spot_check_hashes(
     global_arguments,
     local_path,
     remote_path,
-    log_prefix,
     source_paths,
 ):
     '''
     Given a repository configuration dict, the name of the latest archive, a configuration dict, the
     local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
-    remote Borg path, a log label, and spot check source paths, compare the hashes for a sampling of
+    remote Borg path, and spot check source paths, compare the hashes for a sampling of the source
-    the source paths with hashes from corresponding paths in the given archive. Return a sequence of
+    paths with hashes from corresponding paths in the given archive. Return a sequence of the paths
-    the paths that fail that hash comparison.
+    that fail that hash comparison.
     '''
     # Based on the configured sample percentage, come up with a list of random sample files from the
     # source directories.
     spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
     sample_count = max(
-        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
+        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)),
+        1,
     )
-    source_sample_paths = tuple(random.sample(source_paths, sample_count))
+    source_sample_paths = tuple(random.SystemRandom().sample(source_paths, sample_count))
     working_directory = borgmatic.config.paths.get_working_directory(config)
-    existing_source_sample_paths = {
+    hashable_source_sample_path = {
         source_path
         for source_path in source_sample_paths
-        if os.path.exists(os.path.join(working_directory or '', source_path))
+        for full_source_path in (os.path.join(working_directory or '', source_path),)
+        if os.path.exists(full_source_path)
+        if not os.path.islink(full_source_path)
     }
     logger.debug(
-        f'{log_prefix}: Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
+        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check',
     )
 
     source_sample_paths_iterator = iter(source_sample_paths)
@@ -504,49 +523,102 @@ def compare_spot_check_hashes(
     while True:
         # Hash each file in the sample paths (if it exists).
         source_sample_paths_subset = tuple(
-            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT)
+            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT),
         )
+
         if not source_sample_paths_subset:
             break
 
-        hash_output = borgmatic.execute.execute_command_and_capture_output(
+        hash_paths = tuple(
-            (spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
+            path for path in source_sample_paths_subset if path in hashable_source_sample_path
-            + tuple(
-                path for path in source_sample_paths_subset if path in existing_source_sample_paths
-            ),
-            working_directory=working_directory,
         )
 
-        source_hashes.update(
+        try:
-            **dict(
+            hash_lines = borgmatic.execute.execute_command_and_capture_output(
-                (reversed(line.split('  ', 1)) for line in hash_output.splitlines()),
+                tuple(
-                # Represent non-existent files as having empty hashes so the comparison below still works.
+                    shlex.quote(part)
-                **{
+                    for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
-                    path: ''
+                )
-                    for path in source_sample_paths_subset
+                + hash_paths,
-                    if path not in existing_source_sample_paths
+                working_directory=working_directory,
-                },
             )
-        )
+            source_hashes.update(
+                **dict(
+                    zip(
+                        # xxh64sum rewrites/escapes the paths that it returns alongside its hashes, for
+                        # instance if they contain special characters. When that happens, they don't
+                        # match the original source paths and therefore hash lookups fail. So when
+                        # building this lookup dict, use the original unaltered paths we provided as
+                        # input to xxh64sum.
+                        hash_paths,
+                        (
+                            # For some reason, xxh64sum prefixes the hash with a backslash if the path
+                            # contains a newline. Work around that.
+                            line.split('  ', 1)[0].lstrip('\\')
+                            for line in hash_lines
+                        ),
+                    ),
+                    # Represent non-existent files as having empty hashes so the comparison below still
+                    # works. Same thing for filesystem links, since Borg produces empty archive hashes
+                    # for them.
+                    **{
+                        path: ''
+                        for path in source_sample_paths_subset
+                        if path not in hashable_source_sample_path
+                    },
+                ),
+            )
+        except subprocess.CalledProcessError:
+            # This can happen if a file we planned to hash gets deleted right before we try to hash
+            # it. Falling back to individual file hashing allows us to find and mark just the
+            # file(s) with problems instead of failing the whole batch.
+            logger.warning(
+                'Bulk source path hashing failed for this batch; falling back to individual file hashing'
+            )
+
+            for hash_path in hash_paths:
+                try:
+                    hash_lines = borgmatic.execute.execute_command_and_capture_output(
+                        (
+                            *(
+                                shlex.quote(part)
+                                for part in shlex.split(
+                                    spot_check_config.get('xxh64sum_command', 'xxh64sum')
+                                )
+                            ),
+                            hash_path,
+                        ),
+                        working_directory=working_directory,
+                    )
+                    source_hashes[hash_path] = next(hash_lines).split('  ', 1)[0].lstrip('\\')
+                except (subprocess.CalledProcessError, StopIteration):  # noqa: PERF203
+                    logger.warning(
+                        f'Source path hashing failed for {hash_path}; treating as missing'
+                    )
+                    source_hashes[hash_path] = ''
 
         # Get the hash for each file in the archive.
-        archive_hashes.update(
+        for entry in borgmatic.borg.list.capture_archive_listing(
-            **dict(
+            repository['path'],
-                reversed(line.split(' ', 1))
+            archive,
-                for line in borgmatic.borg.list.capture_archive_listing(
+            config,
-                    repository['path'],
+            local_borg_version,
-                    archive,
+            global_arguments,
-                    config,
+            list_paths=source_sample_paths_subset,
-                    local_borg_version,
+            path_format='{xxh64}{path}{linktarget}{target}',
-                    global_arguments,
+            local_path=local_path,
-                    list_paths=source_sample_paths_subset,
+            remote_path=remote_path,
-                    path_format='{xxh64} {path}{NL}',  # noqa: FS003
+        ):
-                    local_path=local_path,
+            if not entry:
-                    remote_path=remote_path,
+                continue
-                )
+
-                if line
+            # Borg can't get hashes of stored hard links. So if this is a hard link path (and not
-            )
+            # deemed as the "original" by Borg), then skip hashing of it.
-        )
+            if entry.get('linktarget') or entry.get('target'):
+                source_hashes.pop(os.path.join('/', entry['path']), None)
+                continue
+
+            archive_hashes[entry['path']] = entry['xxh64']
 
     # Compare the source hashes with the archive hashes to see how many match.
     failing_paths = []
@@ -562,6 +634,9 @@ def compare_spot_check_hashes(
     return tuple(failing_paths)
 
 
+MAX_SPOT_CHECK_PATHS_LENGTH = 1000
+
+
 def spot_check(
     repository,
     config,
@@ -580,9 +655,6 @@ def spot_check(
     disk to those stored in the latest archive. If any differences are beyond configured tolerances,
     then the check fails.
     '''
-    log_prefix = f'{repository.get("label", repository["path"])}'
-    logger.debug(f'{log_prefix}: Running spot check')
-
     try:
         spot_check_config = next(
             check for check in config.get('checks', ()) if check.get('name') == 'spot'
@@ -592,20 +664,9 @@ def spot_check(
 
     if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
         raise ValueError(
-            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage'
+            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage',
         )
 
-    source_paths = collect_spot_check_source_paths(
-        repository,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-        borgmatic_runtime_directory,
-    )
-    logger.debug(f'{log_prefix}: {len(source_paths)} total source paths for spot check')
-
     archive = borgmatic.borg.repo_list.resolve_archive_name(
         repository['path'],
         'latest',
@@ -615,7 +676,26 @@ def spot_check(
         local_path,
         remote_path,
     )
-    logger.debug(f'{log_prefix}: Using archive {archive} for spot check')
+    logger.debug(f'Using archive {archive} for spot check')
+
+    source_paths = collect_spot_check_source_paths(
+        repository,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        borgmatic_runtime_directory,
+        bootstrap_config_paths=borgmatic.actions.config.bootstrap.load_config_paths_from_archive(
+            repository['path'],
+            archive,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        ),
+    )
+    logger.debug(f'{len(source_paths)} total source paths for spot check')
 
     archive_paths = collect_spot_check_archive_paths(
         repository,
@@ -627,14 +707,17 @@ def spot_check(
         remote_path,
         borgmatic_runtime_directory,
     )
-    logger.debug(f'{log_prefix}: {len(archive_paths)} total archive paths for spot check')
+    logger.debug(f'{len(archive_paths)} total archive paths for spot check')
 
     if len(source_paths) == 0:
-        logger.debug(
+        truncated_archive_paths = textwrap.shorten(
-            f'{log_prefix}: Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}'
+            ', '.join(set(archive_paths)) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
+        logger.debug(f'Paths in latest archive but not source paths: {truncated_archive_paths}')
         raise ValueError(
-            'Spot check failed: There are no source paths to compare against the archive'
+            'Spot check failed; there are no source paths to compare against the archive',
         )
 
     # Calculate the percentage delta between the source paths count and the archive paths count, and
@@ -642,15 +725,19 @@ def spot_check(
     count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
 
     if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
-        rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
+        rootless_source_paths = {path.lstrip(os.path.sep) for path in source_paths}
-        logger.debug(
+        truncated_exclusive_source_paths = textwrap.shorten(
-            f'{log_prefix}: Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}'
+            ', '.join(rootless_source_paths - set(archive_paths)) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
-        logger.debug(
+        truncated_exclusive_archive_paths = textwrap.shorten(
-            f'{log_prefix}: Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}'
+            ', '.join(set(archive_paths) - rootless_source_paths) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
         raise ValueError(
-            f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
+            f'Spot check failed\n{count_delta_percentage:.2f}% file count delta between source paths ({len(source_paths)} total) and latest archive ({len(archive_paths)} total); tolerance is {spot_check_config["count_tolerance_percentage"]}%\nOnly in source paths: {truncated_exclusive_source_paths}\nOnly in latest archive: {truncated_exclusive_archive_paths}',
         )
 
     failing_paths = compare_spot_check_hashes(
@@ -661,25 +748,26 @@ def spot_check(
         global_arguments,
         local_path,
         remote_path,
-        log_prefix,
         source_paths,
     )
 
     # Error if the percentage of failing hashes exceeds the configured tolerance percentage.
-    logger.debug(f'{log_prefix}: {len(failing_paths)} non-matching spot check hashes')
+    logger.debug(f'{len(failing_paths)} non-matching spot check hashes')
     data_tolerance_percentage = spot_check_config['data_tolerance_percentage']
     failing_percentage = (len(failing_paths) / len(source_paths)) * 100
 
     if failing_percentage > data_tolerance_percentage:
-        logger.debug(
+        truncated_failing_paths = textwrap.shorten(
-            f'{log_prefix}: Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
+            ', '.join(failing_paths),
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
         raise ValueError(
-            f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
+            f'Spot check failed\n{failing_percentage:.2f}% of source paths ({len(failing_paths)} out of {len(source_paths)} checked) with data not matching the latest archive; tolerance is {data_tolerance_percentage}%\nSource paths with non-matching data: {truncated_failing_paths}',
         )
 
     logger.info(
-        f'{log_prefix}: Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
+        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta',
     )
 
 
@@ -687,7 +775,6 @@ def run_check(
     config_filename,
     repository,
     config,
-    hook_context,
     local_borg_version,
     check_arguments,
     global_arguments,
@@ -699,22 +786,7 @@ def run_check(
 
     Raise ValueError if the Borg repository ID cannot be determined.
     '''
-    if check_arguments.repository and not borgmatic.config.validate.repositories_match(
+    logger.info('Running consistency checks')
-        repository, check_arguments.repository
-    ):
-        return
-
-    borgmatic.hooks.command.execute_hook(
-        config.get('before_check'),
-        config.get('umask'),
-        config_filename,
-        'pre-check',
-        global_arguments.dry_run,
-        **hook_context,
-    )
-
-    log_prefix = repository.get('label', repository['path'])
-    logger.info(f'{log_prefix}: Running consistency checks')
 
     repository_id = borgmatic.borg.check.get_repository_id(
         repository['path'],
@@ -727,7 +799,10 @@ def run_check(
     upgrade_check_times(config, repository_id)
     configured_checks = parse_checks(config, check_arguments.only_checks)
     archive_filter_flags = borgmatic.borg.check.make_archive_filter_flags(
-        local_borg_version, config, configured_checks, check_arguments
+        local_borg_version,
+        config,
+        configured_checks,
+        check_arguments,
     )
     archives_check_id = make_archives_check_id(archive_filter_flags)
     checks = filter_checks_on_frequency(
@@ -755,6 +830,7 @@ def run_check(
             write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
 
     if 'extract' in checks:
+        logger.info('Running extract check')
         borgmatic.borg.extract.extract_last_archive_dry_run(
             config,
             local_borg_version,
@@ -767,9 +843,8 @@ def run_check(
         write_check_time(make_check_time_path(config, repository_id, 'extract'))
 
     if 'spot' in checks:
-        with borgmatic.config.paths.Runtime_directory(
+        logger.info('Running spot check')
-            config, log_prefix
+        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-        ) as borgmatic_runtime_directory:
             spot_check(
                 repository,
                 config,
@@ -779,13 +854,5 @@ def run_check(
                 remote_path,
                 borgmatic_runtime_directory,
             )
-        write_check_time(make_check_time_path(config, repository_id, 'spot'))
 
-    borgmatic.hooks.command.execute_hook(
+        write_check_time(make_check_time_path(config, repository_id, 'spot'))
-        config.get('after_check'),
-        config.get('umask'),
-        config_filename,
-        'post-check',
-        global_arguments.dry_run,
-        **hook_context,
-    )

borgmatic/actions/compact.py

@@ -2,8 +2,6 @@ import logging
 
 import borgmatic.borg.compact
 import borgmatic.borg.feature
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -12,7 +10,6 @@ def run_compact(
     config_filename,
     repository,
     config,
-    hook_context,
     local_borg_version,
     compact_arguments,
     global_arguments,
@@ -23,23 +20,8 @@ def run_compact(
     '''
     Run the "compact" action for the given repository.
     '''
-    if compact_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, compact_arguments.repository
-    ):
-        return
-
-    borgmatic.hooks.command.execute_hook(
-        config.get('before_compact'),
-        config.get('umask'),
-        config_filename,
-        'pre-compact',
-        global_arguments.dry_run,
-        **hook_context,
-    )
     if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
-        logger.info(
+        logger.info(f'Compacting segments{dry_run_label}')
-            f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}'
-        )
         borgmatic.borg.compact.compact_segments(
             global_arguments.dry_run,
             repository['path'],
@@ -48,19 +30,7 @@ def run_compact(
             global_arguments,
             local_path=local_path,
             remote_path=remote_path,
-            progress=compact_arguments.progress,
             cleanup_commits=compact_arguments.cleanup_commits,
-            threshold=compact_arguments.threshold,
         )
     else:  # pragma: nocover
-        logger.info(
+        logger.info('Skipping compact (only available/needed in Borg 1.2+)')
-            f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)'
-        )
-    borgmatic.hooks.command.execute_hook(
-        config.get('after_compact'),
-        config.get('umask'),
-        config_filename,
-        'post-compact',
-        global_arguments.dry_run,
-        **hook_context,
-    )

borgmatic/actions/config/bootstrap.py

@@ -5,8 +5,6 @@ import os
 import borgmatic.borg.extract
 import borgmatic.borg.repo_list
 import borgmatic.config.paths
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -16,79 +14,81 @@ def make_bootstrap_config(bootstrap_arguments):
     Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
     '''
     return {
-        'ssh_command': bootstrap_arguments.ssh_command,
+        'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory,
+        'local_path': bootstrap_arguments.local_path,
+        'remote_path': bootstrap_arguments.remote_path,
         # In case the repo has been moved or is accessed from a different path at the point of
         # bootstrapping.
         'relocated_repo_access_is_ok': True,
+        'ssh_command': bootstrap_arguments.ssh_command,
+        'user_runtime_directory': bootstrap_arguments.user_runtime_directory,
     }
 
 
-def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
+def load_config_paths_from_archive(
+    repository_path,
+    archive_name,
+    config,
+    local_borg_version,
+    global_arguments,
+    borgmatic_runtime_directory,
+):
     '''
-    Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
+    Given a repository path, an archive name, a configuration dict, the local Borg version, the
-    repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
+    global arguments as an argparse.Namespace, and the borgmatic runtime directory, return the
-    borgmatic source directory, destination directory, and whether to strip components), the global
+    config paths from the manifest.json file in the borgmatic source directory or runtime directory
-    arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
+    within the repository archive.
-    return the config paths from the manifest.json file in the borgmatic source directory or runtime
-    directory after extracting it from the repository archive.
 
     Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
     expected configuration path data.
     '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
-        {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
-    )
-    config = make_bootstrap_config(bootstrap_arguments)
-
     # Probe for the manifest file in multiple locations, as the default location has moved to the
     # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
     # still want to support reading the manifest from previously created archives as well.
-    with borgmatic.config.paths.Runtime_directory(
+    for base_directory in (
-        {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
+        'borgmatic',
-        bootstrap_arguments.repository,
+        borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-    ) as borgmatic_runtime_directory:
+        borgmatic.config.paths.get_borgmatic_source_directory(config),
-        for base_directory in (
+    ):
-            'borgmatic',
+        borgmatic_manifest_path = 'sh:' + os.path.join(
-            borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+            base_directory,
-            borgmatic_source_directory,
+            'bootstrap',
-        ):
+            'manifest.json',
-            borgmatic_manifest_path = 'sh:' + os.path.join(
+        )
-                base_directory, 'bootstrap', 'manifest.json'
-            )
 
-            extract_process = borgmatic.borg.extract.extract_archive(
+        extract_process = borgmatic.borg.extract.extract_archive(
-                global_arguments.dry_run,
+            global_arguments.dry_run,
-                bootstrap_arguments.repository,
+            repository_path,
-                archive_name,
+            archive_name,
-                [borgmatic_manifest_path],
+            [borgmatic_manifest_path],
-                config,
+            config,
-                local_borg_version,
+            local_borg_version,
-                global_arguments,
+            global_arguments,
-                local_path=bootstrap_arguments.local_path,
+            local_path=config.get('local_path', 'borg'),
-                remote_path=bootstrap_arguments.remote_path,
+            remote_path=config.get('remote_path'),
-                extract_to_stdout=True,
+            extract_to_stdout=True,
-            )
+        )
-            manifest_json = extract_process.stdout.read()
+        manifest_json = extract_process.stdout.read()
 
-            if manifest_json:
+        if manifest_json:
-                break
+            break
-        else:
+    else:
-            raise ValueError(
+        raise ValueError(
-                'Cannot read configuration paths from archive due to missing bootstrap manifest'
+            'Cannot read configuration paths from archive due to missing archive or bootstrap manifest',
-            )
+        )
 
     try:
         manifest_data = json.loads(manifest_json)
     except json.JSONDecodeError as error:
         raise ValueError(
-            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
+            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}',
         )
 
     try:
         return manifest_data['config_paths']
     except KeyError:
         raise ValueError(
-            'Cannot read configuration paths from archive due to invalid bootstrap manifest'
+            'Cannot read configuration paths from archive due to invalid bootstrap manifest',
         )
 
 
@@ -109,18 +109,25 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
         local_path=bootstrap_arguments.local_path,
         remote_path=bootstrap_arguments.remote_path,
     )
-    manifest_config_paths = get_config_paths(
-        archive_name, bootstrap_arguments, global_arguments, local_borg_version
-    )
 
-    logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        manifest_config_paths = load_config_paths_from_archive(
+            bootstrap_arguments.repository,
+            archive_name,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        )
+
+    logger.info(f"Bootstrapping configuration paths: {', '.join(manifest_config_paths)}")
 
     borgmatic.borg.extract.extract_archive(
         global_arguments.dry_run,
         bootstrap_arguments.repository,
         archive_name,
         [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
-        config,
+        dict(config, progress=bootstrap_arguments.progress or False),
         local_borg_version,
         global_arguments,
         local_path=bootstrap_arguments.local_path,
@@ -128,5 +135,4 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
         extract_to_stdout=False,
         destination_path=bootstrap_arguments.destination,
         strip_components=bootstrap_arguments.strip_components,
-        progress=bootstrap_arguments.progress,
     )

borgmatic/actions/config/generate.py

@@ -19,30 +19,37 @@ def run_generate(generate_arguments, global_arguments):
     dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
 
     logger.answer(
-        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
+        f'Generating configuration files within: {generate_arguments.destination_path}{dry_run_label}'
+        if generate_arguments.split
+        else f'Generating a configuration file at: {generate_arguments.destination_path}{dry_run_label}'
     )
 
     borgmatic.config.generate.generate_sample_configuration(
         global_arguments.dry_run,
         generate_arguments.source_filename,
-        generate_arguments.destination_filename,
+        generate_arguments.destination_path,
         borgmatic.config.validate.schema_filename(),
         overwrite=generate_arguments.overwrite,
+        split=generate_arguments.split,
     )
 
     if generate_arguments.source_filename:
         logger.answer(
             f'''
-Merged in the contents of configuration file at: {generate_arguments.source_filename}
+Merged the contents of the source configuration file: {generate_arguments.source_filename}'''
-To review the changes made, run:
-
-    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
         )
 
+        if not generate_arguments.split:
+            logger.answer(
+                f'''To review the changes made, run:
+
+    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_path}''',
+            )
+
     logger.answer(
         '''
 This includes all available configuration options with example values, the few
 required options as indicated. Please edit the file to suit your needs.
 
-If you ever need help: https://torsion.org/borgmatic/#issues'''
+If you ever need help: https://torsion.org/borgmatic/#issues''',
     )

borgmatic/actions/config/show.py

@@ -0,0 +1,44 @@
+import json
+import logging
+import sys
+
+import borgmatic.config.generate
+import borgmatic.logger
+
+logger = logging.getLogger(__name__)
+
+
+def run_show(show_arguments, configs):
+    '''
+    Given the show arguments as an argparse.Namespace instance and a dict of configuration filename
+    to corresponding parsed configuration, run the "show" action. That consists of rendering and
+    logging the computed configuration as YAML, separating the configuration for each file with
+    "---".
+
+    If show_arguments.option is set, limit the results to the value of that single option. If
+    show_arguments.json is True, render the results as JSON with one array element per configuration
+    file.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    if show_arguments.json:
+        sys.stdout.write(
+            json.dumps(
+                [
+                    config.get(show_arguments.option) if show_arguments.option else config
+                    for config in configs.values()
+                ]
+            )
+        )
+
+        return
+
+    for config in configs.values():
+        if len(configs) > 1:
+            logger.answer('---')
+
+        logger.answer(
+            borgmatic.config.generate.render_configuration(
+                config.get(show_arguments.option) if show_arguments.option else config
+            ).rstrip()
+        )

borgmatic/actions/config/validate.py

@@ -18,7 +18,7 @@ def run_validate(validate_arguments, configs):
     borgmatic.logger.add_custom_log_levels()
 
     if validate_arguments.show:
-        for config_path, config in configs.items():
+        for config in configs.values():
             if len(configs) > 1:
                 logger.answer('---')
 

borgmatic/actions/create.py

@@ -1,262 +1,23 @@
-import glob
-import itertools
 import logging
-import os
-import pathlib
 
+import borgmatic.actions.dump
 import borgmatic.actions.json
 import borgmatic.borg.create
-import borgmatic.borg.pattern
+import borgmatic.borg.feature
+import borgmatic.borg.rename
+import borgmatic.borg.repo_list
 import borgmatic.config.paths
-import borgmatic.config.validate
-import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
+from borgmatic.actions import pattern
 
 logger = logging.getLogger(__name__)
 
 
-def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
-    '''
-    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
-    return it.
-    '''
-    try:
-        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
-    except ValueError:
-        raise ValueError(f'Invalid pattern: {pattern_line}')
-
-    try:
-        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
-        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
-    except ValueError:
-        pattern_style = default_style
-        path = remainder
-
-    return borgmatic.borg.pattern.Pattern(
-        path,
-        borgmatic.borg.pattern.Pattern_type(pattern_type),
-        borgmatic.borg.pattern.Pattern_style(pattern_style),
-    )
-
-
-def collect_patterns(config):
-    '''
-    Given a configuration dict, produce a single sequence of patterns comprised of the configured
-    source directories, patterns, excludes, pattern files, and exclude files.
-
-    The idea is that Borg has all these different ways of specifying includes, excludes, source
-    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
-    ease of manipulation within borgmatic.
-    '''
-    try:
-        return (
-            tuple(
-                borgmatic.borg.pattern.Pattern(source_directory)
-                for source_directory in config.get('source_directories', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip(), borgmatic.borg.pattern.Pattern_style.SHELL)
-                for pattern_line in config.get('patterns', ())
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for exclude_line in config.get('exclude_patterns', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip(), borgmatic.borg.pattern.Pattern_style.SHELL)
-                for filename in config.get('patterns_from', ())
-                for pattern_line in open(filename).readlines()
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for filename in config.get('exclude_from', ())
-                for exclude_line in open(filename).readlines()
-                if not exclude_line.lstrip().startswith('#')
-                if exclude_line.strip()
-            )
-        )
-    except (FileNotFoundError, OSError) as error:
-        logger.debug(error)
-
-        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
-
-
-def expand_directory(directory, working_directory):
-    '''
-    Given a directory path, expand any tilde (representing a user's home directory) and any globs
-    therein. Return a list of one or more resulting paths.
-
-    Take into account the given working directory so that relative paths are supported.
-    '''
-    expanded_directory = os.path.expanduser(directory)
-
-    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
-    # only available in Python 3.10+.
-    normalized_directory = os.path.join(working_directory or '', expanded_directory)
-    glob_paths = glob.glob(normalized_directory)
-
-    if not glob_paths:
-        return [expanded_directory]
-
-    working_directory_prefix = os.path.join(working_directory or '', '')
-
-    return [
-        (
-            glob_path
-            # If these are equal, that means we didn't add any working directory prefix above.
-            if normalized_directory == expanded_directory
-            # Remove the working directory prefix that we added above in order to make glob() work.
-            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
-            # hack.
-            else glob_path.removeprefix(working_directory_prefix)
-        )
-        for glob_path in glob_paths
-    ]
-
-
-def expand_patterns(patterns, working_directory=None, skip_paths=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    expand tildes and globs in each root pattern. Return all the resulting patterns (not just the
-    root patterns) as a tuple.
-
-    If a set of paths are given to skip, then don't expand any patterns matching them.
-    '''
-    if patterns is None:
-        return ()
-
-    return tuple(
-        itertools.chain.from_iterable(
-            (
-                (
-                    borgmatic.borg.pattern.Pattern(
-                        expanded_path,
-                        pattern.type,
-                        pattern.style,
-                        pattern.device,
-                    )
-                    for expanded_path in expand_directory(pattern.path, working_directory)
-                )
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and pattern.path not in (skip_paths or ())
-                else (pattern,)
-            )
-            for pattern in patterns
-        )
-    )
-
-
-def device_map_patterns(patterns, working_directory=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    determine the identifier for the device on which the pattern's path resides—or None if the path
-    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
-    device field populated. But if the device field is already set, don't bother setting it again.
-
-    This is handy for determining whether two different pattern paths are on the same filesystem
-    (have the same device identifier).
-    '''
-    return tuple(
-        borgmatic.borg.pattern.Pattern(
-            pattern.path,
-            pattern.type,
-            pattern.style,
-            device=pattern.device
-            or (
-                os.stat(full_path).st_dev
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and os.path.exists(full_path)
-                else None
-            ),
-        )
-        for pattern in patterns
-        for full_path in (os.path.join(working_directory or '', pattern.path),)
-    )
-
-
-def deduplicate_patterns(patterns):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
-    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
-    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
-    modification.
-
-    The one exception to deduplication is two paths are on different filesystems (devices). In that
-    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
-    one_file_system option is true).
-
-    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
-    child patterns, because it will naturally spider the contents of the parent pattern's path. And
-    there are cases where Borg coming across the same file twice will result in duplicate reads and
-    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
-    Borg.
-    '''
-    deduplicated = {}  # Use just the keys as an ordered set.
-
-    for pattern in patterns:
-        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-            deduplicated[pattern] = True
-            continue
-
-        parents = pathlib.PurePath(pattern.path).parents
-
-        # If another directory in the given list is a parent of current directory (even n levels up)
-        # and both are on the same filesystem, then the current directory is a duplicate.
-        for other_pattern in patterns:
-            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-                continue
-
-            if any(
-                pathlib.PurePath(other_pattern.path) == parent
-                and pattern.device is not None
-                and other_pattern.device == pattern.device
-                for parent in parents
-            ):
-                break
-        else:
-            deduplicated[pattern] = True
-
-    return tuple(deduplicated.keys())
-
-
-def process_patterns(patterns, working_directory, skip_expand_paths=None):
-    '''
-    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
-    "root" patterns, returning the resulting root and non-root patterns as a list.
-
-    If any paths are given to skip, don't expand them.
-    '''
-    skip_paths = set(skip_expand_paths or ())
-
-    return list(
-        deduplicate_patterns(
-            device_map_patterns(
-                expand_patterns(
-                    patterns,
-                    working_directory=working_directory,
-                    skip_paths=skip_paths,
-                )
-            )
-        )
-    )
-
-
 def run_create(
     config_filename,
     repository,
     config,
     config_paths,
-    hook_context,
     local_borg_version,
     create_arguments,
     global_arguments,
@@ -269,87 +30,161 @@ def run_create(
 
     If create_arguments.json is True, yield the JSON output from creating the archive.
     '''
-    if create_arguments.repository and not borgmatic.config.validate.repositories_match(
+    if config.get('list_details') and config.get('progress'):
-        repository, create_arguments.repository
+        raise ValueError(
-    ):
+            'With the create action, only one of --list/--files/list_details and --progress/progress can be used.',
-        return
+        )
 
-    borgmatic.hooks.command.execute_hook(
+    if config.get('list_details') and create_arguments.json:
-        config.get('before_backup'),
+        raise ValueError(
-        config.get('umask'),
+            'With the create action, only one of --list/--files/list_details and --json can be used.',
-        config_filename,
+        )
-        'pre-backup',
-        global_arguments.dry_run,
-        **hook_context,
-    )
 
-    log_prefix = repository.get('label', repository['path'])
+    logger.info(f'Creating archive{dry_run_label}')
-    logger.info(f'{log_prefix}: Creating archive{dry_run_label}')
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    with borgmatic.config.paths.Runtime_directory(
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-        config, log_prefix
+        patterns = pattern.process_patterns(
-    ) as borgmatic_runtime_directory:
+            pattern.collect_patterns(config, working_directory),
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
             config,
-            repository['path'],
+            working_directory,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
-        patterns = process_patterns(collect_patterns(config), working_directory)
-        active_dumps = borgmatic.hooks.dispatch.call_hooks(
-            'dump_data_sources',
-            config,
-            repository['path'],
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            config_paths,
-            borgmatic_runtime_directory,
-            patterns,
-            global_arguments.dry_run,
         )
 
-        # Process the patterns again in case any data source hooks updated them. Without this step,
+        original_patterns = list(patterns)
-        # we could end up with duplicate paths that cause Borg to hang when it tries to read from
-        # the same named pipe twice.
-        patterns = process_patterns(patterns, working_directory, skip_expand_paths=config_paths)
-        stream_processes = [process for processes in active_dumps.values() for process in processes]
 
-        json_output = borgmatic.borg.create.create_archive(
+        # Use the original patterns so as to disregard any modifications made by any data source
-            global_arguments.dry_run,
+        # hooks, e.g. via dump_data_sources() below.
-            repository['path'],
+        with borgmatic.actions.dump.Dump_cleanup(
-            config,
+            config, borgmatic_runtime_directory, original_patterns, global_arguments.dry_run
-            patterns,
+        ):
-            local_borg_version,
+            active_dumps = borgmatic.hooks.dispatch.call_hooks(
-            global_arguments,
+                'dump_data_sources',
-            borgmatic_runtime_directory,
+                config,
-            local_path=local_path,
+                borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            remote_path=remote_path,
+                config_paths,
-            progress=create_arguments.progress,
+                borgmatic_runtime_directory,
-            stats=create_arguments.stats,
+                patterns,
-            json=create_arguments.json,
+                global_arguments.dry_run,
-            list_files=create_arguments.list_files,
+            )
-            stream_processes=stream_processes,
-        )
 
-        if json_output:
+            # Process the patterns again in case any data source hooks updated them. Without this step,
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+            # we could end up with duplicate paths that cause Borg to hang when it tries to read from
+            # the same named pipe twice.
+            patterns = pattern.process_patterns(
+                patterns,
+                config,
+                working_directory,
+                borgmatic_runtime_directory,
+                skip_expand_paths=config_paths,
+            )
+            stream_processes = [
+                process for processes in active_dumps.values() for process in processes
+            ]
 
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            # If we have stream processes, we first create an archive with .checkpoint suffix. This is
-            'remove_data_source_dumps',
+            # to make sure we only create a real archive if all the streaming processes completed
-            config,
+            # successfully (create_archive will fail if a streaming process fails, but the archive might
-            config_filename,
+            # have already been created at this point).
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            use_checkpoint = bool(stream_processes)
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
-        )
 
-    borgmatic.hooks.command.execute_hook(
+            json_output = borgmatic.borg.create.create_archive(
-        config.get('after_backup'),
+                global_arguments.dry_run,
-        config.get('umask'),
+                repository['path'],
-        config_filename,
+                config,
-        'post-backup',
+                patterns,
+                local_borg_version,
+                global_arguments,
+                borgmatic_runtime_directory,
+                archive_suffix='.checkpoint' if use_checkpoint else '',
+                local_path=local_path,
+                remote_path=remote_path,
+                json=create_arguments.json,
+                comment=create_arguments.comment,
+                stream_processes=stream_processes,
+            )
+
+            if use_checkpoint:
+                rename_checkpoint_archive(
+                    repository['path'],
+                    global_arguments,
+                    config,
+                    local_borg_version,
+                    local_path,
+                    remote_path,
+                )
+
+            if json_output:
+                output = borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+                if use_checkpoint:
+                    # Patch archive name and ID
+                    renamed_archive = borgmatic.borg.repo_list.get_latest_archive(
+                        repository['path'],
+                        config,
+                        local_borg_version,
+                        global_arguments,
+                        local_path,
+                        remote_path,
+                    )
+
+                    output['archive']['name'] = renamed_archive['name']
+                    output['archive']['id'] = renamed_archive['id']
+
+                yield output
+
+
+def rename_checkpoint_archive(
+    repository_path,
+    global_arguments,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    '''
+    Renames the latest archive to not have a '.checkpoint' suffix.
+
+    Raises ValueError if
+    - there is not latest archive
+    - the latest archive does not have a '.checkpoint' suffix
+
+    Implementation note: We cannot reliably get the just created archive name.
+    So we resort to listing the archives and picking the last one.
+
+    A similar comment applies to retrieving the ID of the renamed archive.
+    '''
+    archive = borgmatic.borg.repo_list.get_latest_archive(
+        repository_path,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        consider_checkpoints=True,
+    )
+
+    archive_name = archive['name']
+
+    if not archive_name.endswith('.checkpoint'):
+        raise ValueError(f'Latest archive did not have a .checkpoint suffix. Got: {archive_name}')
+
+    new_archive_name = archive_name.removesuffix('.checkpoint')
+
+    logger.info(f'Renaming archive {archive_name} -> {new_archive_name}')
+
+    borgmatic.borg.rename.rename_archive(
+        repository_path,
+        (
+            archive['id']
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.ARCHIVE_SERIES, local_borg_version
+            )
+            else archive['name']
+        ),
+        new_archive_name,
         global_arguments.dry_run,
-        **hook_context,
+        config,
+        local_borg_version,
+        local_path,
+        remote_path,
     )

borgmatic/actions/delete.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.actions.arguments
 import borgmatic.borg.delete
-import borgmatic.borg.repo_delete
 import borgmatic.borg.repo_list
 
 logger = logging.getLogger(__name__)
@@ -20,31 +19,28 @@ def run_delete(
     '''
     Run the "delete" action for the given repository and archive(s).
     '''
-    if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    logger.answer('Deleting archives')
-        repository, delete_arguments.repository
-    ):
-        logger.answer(f'{repository.get("label", repository["path"])}: Deleting archives')
 
-        archive_name = (
+    archive_name = (
-            borgmatic.borg.repo_list.resolve_archive_name(
+        borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
+            repository['path'],
-                delete_arguments.archive,
+            delete_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            )
-            if delete_arguments.archive
-            else None
-        )
-
-        borgmatic.borg.delete.delete_archives(
-            repository,
             config,
             local_borg_version,
-            borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
             global_arguments,
             local_path,
             remote_path,
         )
+        if delete_arguments.archive
+        else None
+    )
+
+    borgmatic.borg.delete.delete_archives(
+        repository,
+        config,
+        local_borg_version,
+        borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
+        global_arguments,
+        local_path,
+        remote_path,
+    )

borgmatic/actions/diff.py

@@ -0,0 +1,63 @@
+import logging
+
+import borgmatic.actions.pattern
+import borgmatic.borg.diff
+
+logger = logging.getLogger(__name__)
+
+
+def run_diff(
+    repository,
+    config,
+    local_borg_version,
+    diff_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "diff" action for the given repository.
+    '''
+
+    # Only process patterns if only_patterns flag is set
+    if diff_arguments.only_patterns:
+        working_directory = borgmatic.config.paths.get_working_directory(config)
+        processed_patterns = borgmatic.actions.pattern.process_patterns(
+            (*borgmatic.actions.pattern.collect_patterns(config, working_directory),),
+            config,
+            working_directory,
+        )
+    else:
+        processed_patterns = None
+
+    archive = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        diff_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    second_archive = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        diff_arguments.second_archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.borg.diff.diff(
+        repository['path'],
+        archive,
+        second_archive,
+        config,
+        local_borg_version,
+        diff_arguments,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+        patterns=processed_patterns,
+    )

borgmatic/actions/dump.py

@@ -0,0 +1,53 @@
+import borgmatic.hooks.dispatch
+
+
+class Dump_cleanup:
+    '''
+    A Python context manager for removing data source dumps before and after wrapped code. Besides
+    doing this for general filesystem cleanliness reasons, leaving old dumps lying around can
+    prevent future dumping from working (in the case of filesystem snapshots) or cause Borg hangs
+    (in the case of database dump named pipes).
+
+    Example use as a context manager:
+
+        with borgmatic.actions.dump.Dump_cleanup(
+            config, borgmatic_runtime_directory, patterns, dry_run,
+        ):
+            do_something_like_perform_a_dump_or_restore()
+    '''
+
+    def __init__(self, config, borgmatic_runtime_directory, patterns, dry_run):
+        '''
+        Given a configuration dict, the borgmatic runtime directory, the configured patterns, and
+        whether this is a dry-run, store these values for use below.
+        '''
+        self.config = config
+        self.borgmatic_runtime_directory = borgmatic_runtime_directory
+        self.patterns = patterns
+        self.dry_run = dry_run
+
+    def __enter__(self):
+        '''
+        Remove all data source dumps that exist prior to the wrapped code running.
+        '''
+        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            'remove_data_source_dumps',
+            self.config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            self.borgmatic_runtime_directory,
+            self.patterns,
+            self.dry_run,
+        )
+
+    def __exit__(self, exception_type, exception, traceback):
+        '''
+        Remove all data source dumps, including any created by the wrapped code.
+        '''
+        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            'remove_data_source_dumps',
+            self.config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            self.borgmatic_runtime_directory,
+            self.patterns,
+            self.dry_run,
+        )

borgmatic/actions/export_key.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.export_key
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,16 +17,13 @@ def run_export_key(
     '''
     Run the "key export" action for the given repository.
     '''
-    if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    logger.info('Exporting repository key')
-        repository, export_arguments.repository
+    borgmatic.borg.export_key.export_key(
-    ):
+        repository['path'],
-        logger.info(f'{repository.get("label", repository["path"])}: Exporting repository key')
+        config,
-        borgmatic.borg.export_key.export_key(
+        local_borg_version,
-            repository['path'],
+        export_arguments,
-            config,
+        global_arguments,
-            local_borg_version,
+        local_path=local_path,
-            export_arguments,
+        remote_path=remote_path,
-            global_arguments,
+    )
-            local_path=local_path,
-            remote_path=remote_path,
-        )

borgmatic/actions/export_tar.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.borg.export_tar
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -19,32 +18,26 @@ def run_export_tar(
     '''
     Run the "export-tar" action for the given repository.
     '''
-    if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
-        repository, export_tar_arguments.repository
+    borgmatic.borg.export_tar.export_tar_archive(
-    ):
+        global_arguments.dry_run,
-        logger.info(
+        repository['path'],
-            f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file'
+        borgmatic.borg.repo_list.resolve_archive_name(
-        )
-        borgmatic.borg.export_tar.export_tar_archive(
-            global_arguments.dry_run,
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
+            export_tar_arguments.archive,
-                repository['path'],
-                export_tar_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            export_tar_arguments.paths,
-            export_tar_arguments.destination,
             config,
             local_borg_version,
             global_arguments,
-            local_path=local_path,
+            local_path,
-            remote_path=remote_path,
+            remote_path,
-            tar_filter=export_tar_arguments.tar_filter,
+        ),
-            list_files=export_tar_arguments.list_files,
+        export_tar_arguments.paths,
-            strip_components=export_tar_arguments.strip_components,
+        export_tar_arguments.destination,
-        )
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+        tar_filter=export_tar_arguments.tar_filter,
+        strip_components=export_tar_arguments.strip_components,
+    )

borgmatic/actions/extract.py

@@ -2,8 +2,6 @@ import logging
 
 import borgmatic.borg.extract
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -12,7 +10,6 @@ def run_extract(
     config_filename,
     repository,
     config,
-    hook_context,
     local_borg_version,
     extract_arguments,
     global_arguments,
@@ -22,47 +19,25 @@ def run_extract(
     '''
     Run the "extract" action for the given repository.
     '''
-    borgmatic.hooks.command.execute_hook(
+    logger.info(f'Extracting archive {extract_arguments.archive}')
-        config.get('before_extract'),
+    borgmatic.borg.extract.extract_archive(
-        config.get('umask'),
-        config_filename,
-        'pre-extract',
         global_arguments.dry_run,
-        **hook_context,
+        repository['path'],
-    )
+        borgmatic.borg.repo_list.resolve_archive_name(
-    if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, extract_arguments.repository
-    ):
-        logger.info(
-            f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}'
-        )
-        borgmatic.borg.extract.extract_archive(
-            global_arguments.dry_run,
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
+            extract_arguments.archive,
-                repository['path'],
-                extract_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            extract_arguments.paths,
             config,
             local_borg_version,
             global_arguments,
-            local_path=local_path,
+            local_path,
-            remote_path=remote_path,
+            remote_path,
-            destination_path=extract_arguments.destination,
+        ),
-            strip_components=extract_arguments.strip_components,
+        extract_arguments.paths,
-            progress=extract_arguments.progress,
+        config,
-        )
+        local_borg_version,
-    borgmatic.hooks.command.execute_hook(
+        global_arguments,
-        config.get('after_extract'),
+        local_path=local_path,
-        config.get('umask'),
+        remote_path=remote_path,
-        config_filename,
+        destination_path=extract_arguments.destination,
-        'post-extract',
+        strip_components=extract_arguments.strip_components,
-        global_arguments.dry_run,
-        **hook_context,
     )

borgmatic/actions/import_key.py

@@ -0,0 +1,29 @@
+import logging
+
+import borgmatic.borg.import_key
+
+logger = logging.getLogger(__name__)
+
+
+def run_import_key(
+    repository,
+    config,
+    local_borg_version,
+    import_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "key import" action for the given repository.
+    '''
+    logger.info('Importing repository key')
+    borgmatic.borg.import_key.import_key(
+        repository['path'],
+        config,
+        local_borg_version,
+        import_arguments,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/info.py

@@ -4,7 +4,6 @@ import borgmatic.actions.arguments
 import borgmatic.actions.json
 import borgmatic.borg.info
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -23,30 +22,26 @@ def run_info(
 
     If info_arguments.json is True, yield the JSON output from the info for the archive.
     '''
-    if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    if not info_arguments.json:
-        repository, info_arguments.repository
+        logger.answer('Displaying archive summary information')
-    ):
+
-        if not info_arguments.json:
+    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            logger.answer(
+        repository['path'],
-                f'{repository.get("label", repository["path"])}: Displaying archive summary information'
+        info_arguments.archive,
-            )
+        config,
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        local_borg_version,
-            repository['path'],
+        global_arguments,
-            info_arguments.archive,
+        local_path,
-            config,
+        remote_path,
-            local_borg_version,
+    )
-            global_arguments,
+    json_output = borgmatic.borg.info.display_archives_info(
-            local_path,
+        repository['path'],
-            remote_path,
+        config,
-        )
+        local_borg_version,
-        json_output = borgmatic.borg.info.display_archives_info(
+        borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
-            repository['path'],
+        global_arguments,
-            config,
+        local_path,
-            local_borg_version,
+        remote_path,
-            borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
+    )
-            global_arguments,
+    if json_output:
-            local_path,
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
-            remote_path,
-        )
-        if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/list.py

@@ -3,7 +3,6 @@ import logging
 import borgmatic.actions.arguments
 import borgmatic.actions.json
 import borgmatic.borg.list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -22,32 +21,29 @@ def run_list(
 
     If list_arguments.json is True, yield the JSON output from listing the archive.
     '''
-    if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    if not list_arguments.json:
-        repository, list_arguments.repository
+        if list_arguments.find_paths:  # pragma: no cover
-    ):
+            logger.answer('Searching archives')
-        if not list_arguments.json:
+        elif not list_arguments.archive:  # pragma: no cover
-            if list_arguments.find_paths:  # pragma: no cover
+            logger.answer('Listing archives')
-                logger.answer(f'{repository.get("label", repository["path"])}: Searching archives')
-            elif not list_arguments.archive:  # pragma: no cover
-                logger.answer(f'{repository.get("label", repository["path"])}: Listing archives')
 
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            repository['path'],
+        repository['path'],
-            list_arguments.archive,
+        list_arguments.archive,
-            config,
+        config,
-            local_borg_version,
+        local_borg_version,
-            global_arguments,
+        global_arguments,
-            local_path,
+        local_path,
-            remote_path,
+        remote_path,
-        )
+    )
-        json_output = borgmatic.borg.list.list_archive(
+    json_output = borgmatic.borg.list.list_archive(
-            repository['path'],
+        repository['path'],
-            config,
+        config,
-            local_borg_version,
+        local_borg_version,
-            borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
+        borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
-            global_arguments,
+        global_arguments,
-            local_path,
+        local_path,
-            remote_path,
+        remote_path,
-        )
+    )
-        if json_output:
+    if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/mount.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.borg.mount
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -19,31 +18,26 @@ def run_mount(
     '''
     Run the "mount" action for the given repository.
     '''
-    if mount_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    if mount_arguments.archive:
-        repository, mount_arguments.repository
+        logger.info(f'Mounting archive {mount_arguments.archive}')
-    ):
+    else:  # pragma: nocover
-        if mount_arguments.archive:
+        logger.info('Mounting repository')
-            logger.info(
-                f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}'
-            )
-        else:  # pragma: nocover
-            logger.info(f'{repository.get("label", repository["path"])}: Mounting repository')
 
-        borgmatic.borg.mount.mount_archive(
+    borgmatic.borg.mount.mount_archive(
+        repository['path'],
+        borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
+            mount_arguments.archive,
-                repository['path'],
-                mount_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            mount_arguments,
             config,
             local_borg_version,
             global_arguments,
-            local_path=local_path,
+            local_path,
-            remote_path=remote_path,
+            remote_path,
-        )
+        ),
+        mount_arguments,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/pattern.py

@@ -0,0 +1,330 @@
+import glob
+import itertools
+import logging
+import os
+import pathlib
+
+import borgmatic.borg.pattern
+
+logger = logging.getLogger(__name__)
+
+
+def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
+    '''
+    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
+    return it.
+    '''
+    try:
+        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
+    except ValueError:
+        raise ValueError(f'Invalid pattern: {pattern_line}')
+
+    try:
+        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
+        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
+    except ValueError:
+        pattern_style = default_style
+        path = remainder
+
+    return borgmatic.borg.pattern.Pattern(
+        path,
+        borgmatic.borg.pattern.Pattern_type(pattern_type),
+        borgmatic.borg.pattern.Pattern_style(pattern_style),
+        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
+    )
+
+
+def collect_patterns(config, working_directory):
+    '''
+    Given a configuration dict and the working directory, produce a single sequence of patterns
+    comprised of the configured source directories, patterns, excludes, pattern files, and exclude
+    files.
+
+    The idea is that Borg has all these different ways of specifying includes, excludes, source
+    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
+    ease of manipulation within borgmatic.
+    '''
+    try:
+        return (
+            tuple(
+                borgmatic.borg.pattern.Pattern(
+                    source_directory,
+                    source=borgmatic.borg.pattern.Pattern_source.CONFIG,
+                )
+                for source_directory in config.get('source_directories', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip())
+                for pattern_line in config.get('patterns', ())
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for exclude_line in config.get('exclude_patterns', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip())
+                for filename in config.get('patterns_from', ())
+                for expanded_path in expand_directory(filename, working_directory)
+                for pattern_line in open(expanded_path, encoding='utf-8')
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for filename in config.get('exclude_from', ())
+                for expanded_path in expand_directory(filename, working_directory)
+                for exclude_line in open(expanded_path, encoding='utf-8')
+                if not exclude_line.lstrip().startswith('#')
+                if exclude_line.strip()
+            )
+        )
+    except (FileNotFoundError, OSError) as error:
+        logger.debug(error)
+
+        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
+
+
+def expand_directory(directory, working_directory):
+    '''
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+
+    Take into account the given working directory so that relative paths are supported.
+    '''
+    expanded_directory = os.path.expanduser(directory)
+
+    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
+    # only available in Python 3.10+.
+    normalized_directory = os.path.join(working_directory or '', expanded_directory)
+    glob_paths = glob.glob(normalized_directory)
+
+    if not glob_paths:
+        return [expanded_directory]
+
+    working_directory_prefix = os.path.join(working_directory or '', '')
+
+    return [
+        (
+            glob_path
+            # If these are equal, that means we didn't add any working directory prefix above.
+            if normalized_directory == expanded_directory
+            # Remove the working directory prefix added above in order to make glob() work. We
+            # can't use os.path.relpath() here because it collapses any use of Borg's slashdot hack.
+            else glob_path.removeprefix(working_directory_prefix)
+        )
+        for glob_path in glob_paths
+    ]
+
+
+def expand_patterns(patterns, working_directory=None, skip_paths=None):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    expand tildes and globs in each root pattern and expand just tildes in each non-root pattern.
+    The idea is that non-root patterns may be regular expressions or other pattern styles containing
+    "*" that borgmatic should not expand as a shell glob.
+
+    Return all the resulting patterns as a tuple.
+
+    If a set of paths are given to skip, then don't expand any patterns matching them.
+    '''
+    if patterns is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(
+            (
+                (
+                    borgmatic.borg.pattern.Pattern(
+                        expanded_path,
+                        pattern.type,
+                        pattern.style,
+                        pattern.device,
+                        pattern.source,
+                    )
+                    for expanded_path in expand_directory(pattern.path, working_directory)
+                )
+                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.path not in (skip_paths or ())
+                else (
+                    borgmatic.borg.pattern.Pattern(
+                        os.path.expanduser(pattern.path),
+                        pattern.type,
+                        pattern.style,
+                        pattern.device,
+                        pattern.source,
+                    ),
+                )
+            )
+            for pattern in patterns
+        ),
+    )
+
+
+def get_existent_path_or_parent(path):
+    '''
+    Given a path, return it if it exists. Otherwise, return the longest parent directory of the path
+    that exists. Return None if none of these paths exist.
+
+    This is used below for finding an existent path prefix of pattern's path, which is necessary if
+    the path contain globs or other special characters that we don't want to try to interpret
+    (because we want to leave that responsibility to Borg).
+    '''
+    if path.startswith('/e2e/'):
+        return None
+
+    try:
+        return next(
+            candidate_path
+            for candidate_path in (
+                path,
+                *tuple(str(parent) for parent in pathlib.PurePath(path).parents),
+            )
+            if os.path.exists(candidate_path)
+        )
+    except StopIteration:
+        return None
+
+
+def device_map_patterns(patterns, working_directory=None):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    determine the identifier for the device on which the pattern's path resides—or None if the path
+    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
+    device field populated. But if the device field is already set, don't bother setting it again.
+
+    This is handy for determining whether two different pattern paths are on the same filesystem
+    (have the same device identifier).
+
+    This function only considers the start of a pattern's path—from the start of the path up until
+    there's a path component with a glob or other non-literal character. If there are no such
+    characters, the whole path is considered. The rationale is that it's not feasible for borgmatic
+    to interpret Borg's patterns to see which actual files (and therefore devices) they map to. So
+    for instance, a pattern with a path of "/var/log/*/data" would end up with its device set to the
+    device of "/var/log"—ignoring the "/*/data" part due to that glob.
+
+    The one exception is that if a regular expression pattern path starts with "^", that will get
+    stripped off for purposes of determining its device.
+    '''
+    return tuple(
+        borgmatic.borg.pattern.Pattern(
+            pattern.path,
+            pattern.type,
+            pattern.style,
+            device=pattern.device or (os.stat(existent_path).st_dev if existent_path else None),
+            source=pattern.source,
+        )
+        for pattern in patterns
+        for existent_path in (
+            get_existent_path_or_parent(
+                os.path.join(working_directory or '', pattern.path.lstrip('^')),
+            ),
+        )
+    )
+
+
+def deduplicate_runtime_directory_patterns(patterns, config, borgmatic_runtime_directory=None):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, the borgmatic runtime directory,
+    and a configuration dict, return them without any duplicate root child patterns that contain the
+    runtime directory. For instance, if two root patterns are given with paths "/foo" and
+    "/foo/bar", and the runtime directory is "/foo/bar", return just the "/foo" pattern. Non-root
+    patterns and patterns not containing the runtime directory are passed through without
+    modification.
+
+    One exception to deduplication is if two paths are on different filesystems (devices) and
+    "one_file_system" is True in the given configuration. In that case, the paths won't get
+    deduplicated, because Borg won't cross filesystem boundaries when "one_file_system" is True.
+
+    The idea is that if Borg is given a root parent pattern containing the borgmatic runtime
+    directory, then Borg doesn't also need to be given child patterns, because it will naturally
+    spider the contents of the parent pattern's path. Additionally, there are cases where Borg
+    coming across the same file twice will result in duplicate reads and even hangs, e.g. when a
+    database hook in the borgmatic runtime directory is using a named pipe for streaming database
+    dumps to Borg.
+
+    This deduplication is limited to the borgmatic runtime directory (where borgmatic's named pipes
+    exist), because there are other legitimate use cases for parent and child patterns to both exist
+    in patterns. For instance, with some snapshotted filesystems, snapshots don't traverse from a
+    parent filesystem to a child and therefore both need to remain in patterns.
+
+    And for the case of named pipes outside of the borgmatic runtime directory, there is code
+    elsewhere (in the "create" action) that auto-excludes special files to prevent Borg hangs.
+    '''
+    if borgmatic_runtime_directory is None:
+        return patterns
+
+    deduplicated = {}  # Use just the keys as an ordered set.
+    runtime_directory_parents = set(pathlib.PurePath(borgmatic_runtime_directory).parents).union(
+        {pathlib.PurePath(borgmatic_runtime_directory)}
+    )
+
+    for pattern in patterns:
+        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+            deduplicated[pattern] = True
+            continue
+
+        pattern_parents = pathlib.PurePath(pattern.path).parents
+
+        # If:
+        #
+        #   1. another pattern is a parent of the current pattern (even n levels up),
+        #   2. both patterns are parents of the runtime directory (even n levels up),
+        #   3. and both patterns are on the same filesystem (or one_file_system is not set)
+        #
+        # ... then consider the current pattern as a duplicate.
+        for other_pattern in patterns:
+            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+                continue
+
+            device_matches = pattern.device is not None and (
+                other_pattern.device == pattern.device or config.get('one_file_system') is not True
+            )
+
+            if (
+                pathlib.PurePath(other_pattern.path) in pattern_parents
+                and pathlib.PurePosixPath(other_pattern.path) in runtime_directory_parents
+                and pathlib.PurePosixPath(pattern.path) in runtime_directory_parents
+                and device_matches
+            ):
+                break
+        else:
+            deduplicated[pattern] = True
+
+    return tuple(deduplicated.keys())
+
+
+def process_patterns(
+    patterns, config, working_directory, borgmatic_runtime_directory=None, skip_expand_paths=None
+):
+    '''
+    Given a sequence of Borg patterns, a configuration dict, a configured working directory, the
+    borgmatic runtime directory, and a sequence of paths to skip path expansion for, expand and
+    deduplicate any "root" patterns, returning the resulting root and non-root patterns as a list.
+
+    If the borgmatic runtime directory is None, then don't deduplicate patterns. Deduplication is
+    really only necessary for the "create" action when the runtime directory might contain named
+    pipes for database dumps.
+    '''
+    skip_paths = set(skip_expand_paths or ())
+
+    return list(
+        deduplicate_runtime_directory_patterns(
+            device_map_patterns(
+                expand_patterns(
+                    patterns,
+                    working_directory=working_directory,
+                    skip_paths=skip_paths,
+                ),
+            ),
+            config,
+            borgmatic_runtime_directory,
+        ),
+    )

borgmatic/actions/prune.py

@@ -1,8 +1,6 @@
 import logging
 
 import borgmatic.borg.prune
-import borgmatic.config.validate
-import borgmatic.hooks.command
 
 logger = logging.getLogger(__name__)
 
@@ -11,7 +9,6 @@ def run_prune(
     config_filename,
     repository,
     config,
-    hook_context,
     local_borg_version,
     prune_arguments,
     global_arguments,
@@ -22,20 +19,7 @@ def run_prune(
     '''
     Run the "prune" action for the given repository.
     '''
-    if prune_arguments.repository and not borgmatic.config.validate.repositories_match(
+    logger.info(f'Pruning archives{dry_run_label}')
-        repository, prune_arguments.repository
-    ):
-        return
-
-    borgmatic.hooks.command.execute_hook(
-        config.get('before_prune'),
-        config.get('umask'),
-        config_filename,
-        'pre-prune',
-        global_arguments.dry_run,
-        **hook_context,
-    )
-    logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}')
     borgmatic.borg.prune.prune_archives(
         global_arguments.dry_run,
         repository['path'],
@@ -46,11 +30,3 @@ def run_prune(
         local_path=local_path,
         remote_path=remote_path,
     )
-    borgmatic.hooks.command.execute_hook(
-        config.get('after_prune'),
-        config.get('umask'),
-        config_filename,
-        'post-prune',
-        global_arguments.dry_run,
-        **hook_context,
-    )

borgmatic/actions/recreate.py

@@ -0,0 +1,94 @@
+import logging
+import subprocess
+
+import borgmatic.actions.pattern
+import borgmatic.borg.pattern
+import borgmatic.borg.recreate
+import borgmatic.borg.repo_list
+
+logger = logging.getLogger(__name__)
+
+
+BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS = 30
+
+
+def run_recreate(
+    repository,
+    config,
+    local_borg_version,
+    recreate_arguments,
+    global_arguments,
+    dry_run_label,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "recreate" action for the given repository.
+    '''
+    if recreate_arguments.archive:
+        logger.answer(f'Recreating archive {recreate_arguments.archive}{dry_run_label}')
+    else:
+        logger.answer(f'Recreating repository{dry_run_label}')
+
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    # Collect and process patterns.
+    processed_patterns = borgmatic.actions.pattern.process_patterns(
+        (
+            *borgmatic.actions.pattern.collect_patterns(config, working_directory),
+            # Also add borgmatic-specific paths, so they don't get excluded from the recreated
+            # archive. Note that this doesn't currently work for archives created with Borg 1.2 or
+            # below.
+            borgmatic.borg.pattern.Pattern(
+                '/borgmatic', source=borgmatic.borg.pattern.Pattern_source.INTERNAL
+            ),
+        ),
+        config,
+        working_directory,
+    )
+
+    archive = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        recreate_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    if archive and archive.endswith('.recreate'):
+        if recreate_arguments.archive == 'latest':
+            raise ValueError(
+                f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.',
+            )
+
+        raise ValueError(
+            f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.',
+        )
+
+    try:
+        borgmatic.borg.recreate.recreate_archive(
+            repository['path'],
+            archive,
+            config,
+            local_borg_version,
+            recreate_arguments,
+            global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+            patterns=processed_patterns,
+        )
+    except subprocess.CalledProcessError as error:
+        if error.returncode == BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS:
+            if recreate_arguments.target:
+                raise ValueError(
+                    f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.',
+                )
+
+            if archive:
+                raise ValueError(
+                    f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.',
+                )
+
+        raise

borgmatic/actions/repo_create.py

@@ -1,7 +1,6 @@
 import logging
 
 import borgmatic.borg.repo_create
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -18,24 +17,39 @@ def run_repo_create(
     '''
     Run the "repo-create" action for the given repository.
     '''
-    if repo_create_arguments.repository and not borgmatic.config.validate.repositories_match(
+    logger.info('Creating repository')
-        repository, repo_create_arguments.repository
+
-    ):
+    encryption_mode = repo_create_arguments.encryption_mode or repository.get('encryption')
-        return
+
+    if not encryption_mode:
+        raise ValueError(
+            'With the repo-create action, either the --encryption flag or the repository encryption option is required.',
+        )
 
-    logger.info(f'{repository.get("label", repository["path"])}: Creating repository')
     borgmatic.borg.repo_create.create_repository(
         global_arguments.dry_run,
         repository['path'],
         config,
         local_borg_version,
         global_arguments,
-        repo_create_arguments.encryption_mode,
+        encryption_mode,
         repo_create_arguments.source_repository,
         repo_create_arguments.copy_crypt_key,
-        repo_create_arguments.append_only,
+        (
-        repo_create_arguments.storage_quota,
+            repository.get('append_only')
-        repo_create_arguments.make_parent_dirs,
+            if repo_create_arguments.append_only is None
+            else repo_create_arguments.append_only
+        ),
+        (
+            repository.get('storage_quota')
+            if repo_create_arguments.storage_quota is None
+            else repo_create_arguments.storage_quota
+        ),
+        (
+            repository.get('make_parent_directories')
+            if repo_create_arguments.make_parent_directories is None
+            else repo_create_arguments.make_parent_directories
+        ),
         local_path=local_path,
         remote_path=remote_path,
     )

borgmatic/actions/repo_delete.py

@@ -17,20 +17,16 @@ def run_repo_delete(
     '''
     Run the "repo-delete" action for the given repository.
     '''
-    if repo_delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    logger.answer(
-        repository, repo_delete_arguments.repository
+        'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else ''),
-    ):
+    )
-        logger.answer(
-            f'{repository.get("label", repository["path"])}: Deleting repository'
-            + (' cache' if repo_delete_arguments.cache_only else '')
-        )
 
-        borgmatic.borg.repo_delete.delete_repository(
+    borgmatic.borg.repo_delete.delete_repository(
-            repository,
+        repository,
-            config,
+        config,
-            local_borg_version,
+        local_borg_version,
-            repo_delete_arguments,
+        repo_delete_arguments,
-            global_arguments,
+        global_arguments,
-            local_path,
+        local_path,
-            remote_path,
+        remote_path,
-        )
+    )

borgmatic/actions/repo_info.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.actions.json
 import borgmatic.borg.repo_info
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -21,22 +20,17 @@ def run_repo_info(
 
     If repo_info_arguments.json is True, yield the JSON output from the info for the repository.
     '''
-    if repo_info_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    if not repo_info_arguments.json:
-        repository, repo_info_arguments.repository
+        logger.answer('Displaying repository summary information')
-    ):
-        if not repo_info_arguments.json:
-            logger.answer(
-                f'{repository.get("label", repository["path"])}: Displaying repository summary information'
-            )
 
-        json_output = borgmatic.borg.repo_info.display_repository_info(
+    json_output = borgmatic.borg.repo_info.display_repository_info(
-            repository['path'],
+        repository['path'],
-            config,
+        config,
-            local_borg_version,
+        local_borg_version,
-            repo_info_arguments=repo_info_arguments,
+        repo_info_arguments=repo_info_arguments,
-            global_arguments=global_arguments,
+        global_arguments=global_arguments,
-            local_path=local_path,
+        local_path=local_path,
-            remote_path=remote_path,
+        remote_path=remote_path,
-        )
+    )
-        if json_output:
+    if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/repo_list.py

@@ -2,7 +2,6 @@ import logging
 
 import borgmatic.actions.json
 import borgmatic.borg.repo_list
-import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
 
@@ -21,20 +20,17 @@ def run_repo_list(
 
     If repo_list_arguments.json is True, yield the JSON output from listing the repository.
     '''
-    if repo_list_arguments.repository is None or borgmatic.config.validate.repositories_match(
+    if not repo_list_arguments.json:
-        repository, repo_list_arguments.repository
+        logger.answer('Listing repository')
-    ):
-        if not repo_list_arguments.json:
-            logger.answer(f'{repository.get("label", repository["path"])}: Listing repository')
 
-        json_output = borgmatic.borg.repo_list.list_repository(
+    json_output = borgmatic.borg.repo_list.list_repository(
-            repository['path'],
+        repository['path'],
-            config,
+        config,
-            local_borg_version,
+        local_borg_version,
-            repo_list_arguments=repo_list_arguments,
+        repo_list_arguments=repo_list_arguments,
-            global_arguments=global_arguments,
+        global_arguments=global_arguments,
-            local_path=local_path,
+        local_path=local_path,
-            remote_path=remote_path,
+        remote_path=remote_path,
-        )
+    )
-        if json_output:
+    if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+        yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/restore.py

@@ -1,16 +1,17 @@
 import collections
+import locale
 import logging
 import os
 import pathlib
 import shutil
 import tempfile
 
+import borgmatic.actions.dump
+import borgmatic.actions.pattern
 import borgmatic.borg.extract
 import borgmatic.borg.list
-import borgmatic.borg.mount
 import borgmatic.borg.repo_list
 import borgmatic.config.paths
-import borgmatic.config.validate
 import borgmatic.hooks.data_source.dump
 import borgmatic.hooks.dispatch
 
@@ -22,8 +23,8 @@ UNSPECIFIED = object()
 
 Dump = collections.namedtuple(
     'Dump',
-    ('hook_name', 'data_source_name', 'hostname', 'port'),
+    ('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'),
-    defaults=('localhost', None),
+    defaults=(None, None, None, None),
 )
 
 
@@ -33,17 +34,24 @@ def dumps_match(first, second, default_port=None):
     indicates that the field should match any value. If a default port is given, then consider any
     dump having that port to match with a dump having a None port.
     '''
-    for field_name in first._fields:
+    # label kinda counts as an unique id, if they match ignore host/container/port
+    if first.label not in {None, UNSPECIFIED} and first.label == second.label:
+        field_list = ('hook_name', 'data_source_name')
+    else:
+        field_list = Dump._fields
+
+    for field_name in field_list:
         first_value = getattr(first, field_name)
         second_value = getattr(second, field_name)
 
         if default_port is not None and field_name == 'port':
             if first_value == default_port and second_value is None:
                 continue
+
             if second_value == default_port and first_value is None:
                 continue
 
-        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
+        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:  # noqa: PLR1714
             continue
 
         if first_value != second_value:
@@ -56,25 +64,28 @@ def render_dump_metadata(dump):
     '''
     Given a Dump instance, make a display string describing it for use in log messages.
     '''
+    label = dump.label or UNSPECIFIED
     name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
-    hostname = dump.hostname or 'localhost'
+    host = dump.container or dump.hostname or UNSPECIFIED
     port = None if dump.port is UNSPECIFIED else dump.port
 
-    if port:
+    if label is not UNSPECIFIED:
-        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
+        metadata = f'{name}@{label}'
+    elif port:
+        metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}'
     else:
-        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
+        metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}'
 
-    if dump.hook_name not in (None, UNSPECIFIED):
+    if dump.hook_name not in {None, UNSPECIFIED}:
         return f'{metadata} ({dump.hook_name})'
 
     return metadata
 
 
-def get_configured_data_source(config, restore_dump, log_prefix):
+def get_configured_data_source(config, restore_dump):
     '''
     Search in the given configuration dict for dumps corresponding to the given dump to restore. If
-    there are multiple matches, error. Log using the given log prefix.
+    there are multiple matches, error.
 
     Return the found data source as a data source configuration dict or None if not found.
     '''
@@ -91,7 +102,6 @@ def get_configured_data_source(config, restore_dump, log_prefix):
             borgmatic.hooks.dispatch.call_hook(
                 function_name='get_default_port',
                 config=config,
-                log_prefix=log_prefix,
                 hook_name=hook_name,
             ),
         )
@@ -99,8 +109,10 @@ def get_configured_data_source(config, restore_dump, log_prefix):
             Dump(
                 hook_name,
                 hook_data_source.get('name'),
-                hook_data_source.get('hostname', 'localhost'),
+                hook_data_source.get('hostname'),
                 hook_data_source.get('port'),
+                hook_data_source.get('label') or UNSPECIFIED,
+                hook_data_source.get('container'),
             ),
             restore_dump,
             default_port,
@@ -112,20 +124,21 @@ def get_configured_data_source(config, restore_dump, log_prefix):
 
     if len(matching_dumps) > 1:
         raise ValueError(
-            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
+            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured',
         )
 
     return matching_dumps[0]
 
 
 def strip_path_prefix_from_extracted_dump_destination(
-    destination_path, borgmatic_runtime_directory
+    extract_path,
+    borgmatic_runtime_directory,
 ):
     '''
     Directory-format dump files get extracted into a temporary directory containing a path prefix
-    that depends how the files were stored in the archive. So, given the destination path where the
+    that depends how the files were stored in the archive. So, given the path where the dump was
-    dump was extracted and the borgmatic runtime directory, move the dump files such that the
+    extracted and the borgmatic runtime directory, move the dump files such that the restore doesn't
-    restore doesn't have to deal with that varying path prefix.
+    have to deal with that varying path prefix.
 
     For instance, if the dump was extracted to:
 
@@ -139,15 +152,16 @@ def strip_path_prefix_from_extracted_dump_destination(
 
       /run/user/0/borgmatic/postgresql_databases/test/...
     '''
-    for subdirectory_path, _, _ in os.walk(destination_path):
+    for subdirectory_path, _, _ in os.walk(extract_path):
         databases_directory = os.path.basename(subdirectory_path)
 
         if not databases_directory.endswith('_databases'):
             continue
 
-        shutil.move(
+        destination_path = os.path.join(borgmatic_runtime_directory, databases_directory)
-            subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)
+        shutil.rmtree(destination_path, ignore_errors=True)
-        )
+        shutil.move(subdirectory_path, destination_path)
+
         break
 
 
@@ -170,20 +184,28 @@ def restore_single_dump(
     that data source from the archive.
     '''
     dump_metadata = render_dump_metadata(
-        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
+        Dump(
+            hook_name,
+            data_source['name'],
+            data_source.get('hostname'),
+            data_source.get('port'),
+            data_source.get('label') or UNSPECIFIED,
+            data_source.get('container'),
+        ),
     )
 
-    logger.info(
+    logger.info(f'Restoring data source {dump_metadata}')
-        f'{repository.get("label", repository["path"])}: Restoring data source {dump_metadata}'
-    )
 
     dump_patterns = borgmatic.hooks.dispatch.call_hooks(
         'make_data_source_dump_patterns',
         config,
-        repository['path'],
         borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
         borgmatic_runtime_directory,
         data_source['name'],
+        data_source.get('hostname'),
+        data_source.get('port'),
+        data_source.get('container'),
+        data_source.get('label'),
     )[hook_name.split('_databases', 1)[0]]
 
     destination_path = (
@@ -201,8 +223,8 @@ def restore_single_dump(
             archive=archive_name,
             paths=[
                 borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
-                    dump_patterns
+                    dump_patterns,
-                )
+                ),
             ],
             config=config,
             local_borg_version=local_borg_version,
@@ -217,7 +239,8 @@ def restore_single_dump(
 
         if destination_path and not global_arguments.dry_run:
             strip_path_prefix_from_extracted_dump_destination(
-                destination_path, borgmatic_runtime_directory
+                destination_path,
+                borgmatic_runtime_directory,
             )
     finally:
         if destination_path and not global_arguments.dry_run:
@@ -227,7 +250,6 @@ def restore_single_dump(
     borgmatic.hooks.dispatch.call_hook(
         function_name='restore_data_source_dump',
         config=config,
-        log_prefix=repository['path'],
         hook_name=hook_name,
         data_source=data_source,
         dry_run=global_arguments.dry_run,
@@ -251,16 +273,12 @@ def collect_dumps_from_archive(
     Given a local or remote repository path, a resolved archive name, a configuration dict, the
     local Borg version, global arguments an argparse.Namespace, local and remote Borg paths, and the
     borgmatic runtime directory, query the archive for the names of data sources dumps it contains
-    and return them as a set of Dump instances.
+    and return them as a tuple of Dump instances.
     '''
-    borgmatic_source_directory = str(
+    dumps_from_archive = {}  # Use a dict as an ordered set.
-        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
-    )
 
-    # Probe for the data source dumps in multiple locations, as the default location has moved to
+    # There is (at most) one dump metadata file per data source hook. Load each.
-    # the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
+    for dumps_metadata_entry in borgmatic.borg.list.capture_archive_listing(
-    # we still want to support reading dumps from previously created archives as well.
-    dump_paths = borgmatic.borg.list.capture_archive_listing(
         repository,
         archive,
         config,
@@ -269,7 +287,69 @@ def collect_dumps_from_archive(
         list_paths=[
             'sh:'
             + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
-                base_directory, '*_databases/*/*'
+                base_directory,
+                '*_databases/dumps.json',
+            )
+            # Probe for dump metadata files in multiple locations, as the default location is
+            # "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the
+            # borgmatic runtime directory for older versions of Borg.
+            for base_directory in (
+                'borgmatic',
+                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+            )
+        ],
+        local_path=local_path,
+        remote_path=remote_path,
+    ):
+        if not dumps_metadata_entry.get('path'):
+            continue
+
+        for dump in borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata(
+            borgmatic.borg.extract.extract_archive(
+                global_arguments.dry_run,
+                repository,
+                archive,
+                [dumps_metadata_entry['path']],
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path=local_path,
+                remote_path=remote_path,
+                extract_to_stdout=True,
+            )
+            .stdout.read()
+            .decode(locale.getpreferredencoding()),
+            dumps_metadata_entry['path'],
+        ):
+            dumps_from_archive[dump] = None
+
+    # If we've successfully loaded any dumps metadata, we're done.
+    if dumps_from_archive:
+        logger.debug('Collecting database dumps from archive data source dumps metadata files')
+
+        return tuple(dumps_from_archive.keys())
+
+    # No dumps metadata files were found, so for backwards compatibility, fall back to parsing the
+    # paths of dumps found in the archive to get their respective dump metadata.
+    logger.debug('Collecting database dumps from archive data source dump paths (fallback)')
+    borgmatic_source_directory = str(
+        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)),
+    )
+
+    # Probe for the data source dumps in multiple locations, as the default location has moved to
+    # the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
+    # we still want to support reading dumps from previously created archives as well.
+    dump_entries = borgmatic.borg.list.capture_archive_listing(
+        repository,
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        list_paths=[
+            'sh:'
+            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
+                base_directory,
+                '*_databases/*/*',
             )
             for base_directory in (
                 'borgmatic',
@@ -281,10 +361,9 @@ def collect_dumps_from_archive(
         remote_path=remote_path,
     )
 
-    # Parse the paths of dumps found in the archive to get their respective dump metadata.
+    for dump_entry in dump_entries:
-    dumps_from_archive = set()
+        dump_path = dump_entry.get('path')
 
-    for dump_path in dump_paths:
         if not dump_path:
             continue
 
@@ -296,7 +375,8 @@ def collect_dumps_from_archive(
         ):
             try:
                 (hook_name, host_and_port, data_source_name) = dump_path.split(
-                    base_directory + os.path.sep, 1
+                    base_directory + os.path.sep,
+                    1,
                 )[1].split(os.path.sep)[0:3]
             except (ValueError, IndexError):
                 continue
@@ -313,29 +393,34 @@ def collect_dumps_from_archive(
             except (ValueError, TypeError):
                 port = None
 
-            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
+            dumps_from_archive[
+                Dump(
+                    hook_name, data_source_name, None if hostname == 'localhost' else hostname, port
+                )
+            ] = None
 
             # We've successfully parsed the dump path, so need to probe any further.
             break
         else:
             logger.warning(
-                f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
+                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}',
             )
 
-    return dumps_from_archive
+    return tuple(dumps_from_archive.keys())
 
 
 def get_dumps_to_restore(restore_arguments, dumps_from_archive):
     '''
     Given restore arguments as an argparse.Namespace instance indicating which dumps to restore and
-    a set of Dump instances representing the dumps found in an archive, return a set of specific
+    a tuple of Dump instances representing the dumps found in an archive, return a tuple of specific
     Dump instances from the archive to restore. As part of this, replace any Dump having a data
     source name of "all" with multiple named Dump instances as appropriate.
 
     Raise ValueError if any of the requested data source names cannot be found in the archive or if
     there are multiple archive dump matches for a given requested dump.
     '''
-    requested_dumps = (
+    requested_dumps = tuple(
+        # Use a dict comprehension as an ordered set.
         {
             Dump(
                 hook_name=(
@@ -348,27 +433,37 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
                     else UNSPECIFIED
                 ),
                 data_source_name=name,
-                hostname=restore_arguments.original_hostname or 'localhost',
+                hostname=restore_arguments.original_hostname or UNSPECIFIED,
                 port=restore_arguments.original_port,
-            )
+                label=restore_arguments.original_label or UNSPECIFIED,
-            for name in restore_arguments.data_sources
+                container=restore_arguments.original_container or UNSPECIFIED,
-        }
+            ): None
-        if restore_arguments.data_sources
+            for name in restore_arguments.data_sources or (UNSPECIFIED,)
-        else {
+        }.keys()
+        if restore_arguments.hook
+        or restore_arguments.data_sources
+        or restore_arguments.original_hostname
+        or restore_arguments.original_port
+        or restore_arguments.original_label
+        or restore_arguments.original_container
+        else (
             Dump(
                 hook_name=UNSPECIFIED,
                 data_source_name='all',
                 hostname=UNSPECIFIED,
                 port=UNSPECIFIED,
-            )
+                label=UNSPECIFIED,
-        }
+                container=UNSPECIFIED,
+            ),
+        )
     )
     missing_dumps = set()
-    dumps_to_restore = set()
+    dumps_to_restore = {}  # Use a dict as an ordered set.
 
     # If there's a requested "all" dump, add every dump from the archive to the dumps to restore.
     if any(dump for dump in requested_dumps if dump.data_source_name == 'all'):
-        dumps_to_restore.update(dumps_from_archive)
+        for dump in dumps_from_archive:
+            dumps_to_restore[dump] = None
 
     # If any archive dump matches a requested dump, add the archive dump to the dumps to restore.
     for requested_dump in requested_dumps:
@@ -384,10 +479,10 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
         if len(matching_dumps) == 0:
             missing_dumps.add(requested_dump)
         elif len(matching_dumps) == 1:
-            dumps_to_restore.add(matching_dumps[0])
+            dumps_to_restore[matching_dumps[0]] = None
         else:
             raise ValueError(
-                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
+                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.',
             )
 
     if missing_dumps:
@@ -396,30 +491,31 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
         )
 
         raise ValueError(
-            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
+            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive",
         )
 
-    return dumps_to_restore
+    return tuple(dumps_to_restore.keys())
 
 
 def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
     '''
-    Given a set of requested dumps to restore and a set of dumps actually restored, raise ValueError
+    Given a tuple of requested dumps to restore and a set of dumps actually restored, raise
-    if any requested dumps to restore weren't restored, indicating that they were missing from the
+    ValueError if any requested dumps to restore weren't restored, indicating that they were missing
-    configuration.
+    from the configuration.
     '''
-    if not dumps_actually_restored:
+    if not dumps_to_restore:
         raise ValueError('No data source dumps were found to restore')
 
     missing_dumps = sorted(
-        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
+        set(dumps_to_restore) - dumps_actually_restored,
+        key=lambda dump: dump.data_source_name,
     )
 
     if missing_dumps:
         rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
 
         raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
+            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration",
         )
 
 
@@ -433,108 +529,96 @@ def run_restore(
     remote_path,
 ):
     '''
-    Run the "restore" action for the given repository, but only if the repository matches the
+    Run the "restore" action for the given repository.
-    requested repository in restore arguments.
 
     Raise ValueError if a configured data source could not be found to restore or there's no
     matching dump in the archive.
     '''
-    if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
+    logger.info(f'Restoring data sources from archive {restore_arguments.archive}')
-        repository, restore_arguments.repository
+    working_directory = borgmatic.config.paths.get_working_directory(config)
-    ):
-        return
 
-    log_prefix = repository.get('label', repository['path'])
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-    logger.info(f'{log_prefix}: Restoring data sources from archive {restore_arguments.archive}')
+        patterns = borgmatic.actions.pattern.process_patterns(
-
+            borgmatic.actions.pattern.collect_patterns(config, working_directory),
-    with borgmatic.config.paths.Runtime_directory(
-        config, log_prefix
-    ) as borgmatic_runtime_directory:
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-            'remove_data_source_dumps',
             config,
-            repository['path'],
+            working_directory,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-            borgmatic_runtime_directory,
-            global_arguments.dry_run,
         )
 
-        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        with borgmatic.actions.dump.Dump_cleanup(
-            repository['path'],
+            config, borgmatic_runtime_directory, patterns, global_arguments.dry_run
-            restore_arguments.archive,
+        ):
-            config,
+            archive_name = borgmatic.borg.repo_list.resolve_archive_name(
-            local_borg_version,
+                repository['path'],
-            global_arguments,
+                restore_arguments.archive,
-            local_path,
-            remote_path,
-        )
-        dumps_from_archive = collect_dumps_from_archive(
-            repository['path'],
-            archive_name,
-            config,
-            local_borg_version,
-            global_arguments,
-            local_path,
-            remote_path,
-            borgmatic_runtime_directory,
-        )
-        dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
-
-        dumps_actually_restored = set()
-        connection_params = {
-            'hostname': restore_arguments.hostname,
-            'port': restore_arguments.port,
-            'username': restore_arguments.username,
-            'password': restore_arguments.password,
-            'restore_path': restore_arguments.restore_path,
-        }
-
-        # Restore each dump.
-        for restore_dump in dumps_to_restore:
-            found_data_source = get_configured_data_source(
-                config,
-                restore_dump,
-                log_prefix=repository['path'],
-            )
-
-            # For a dump that wasn't found via an exact match in the configuration, try to fallback
-            # to an "all" data source.
-            if not found_data_source:
-                found_data_source = get_configured_data_source(
-                    config,
-                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
-                    log_prefix=repository['path'],
-                )
-
-                if not found_data_source:
-                    continue
-
-                found_data_source = dict(found_data_source)
-                found_data_source['name'] = restore_dump.data_source_name
-
-            dumps_actually_restored.add(restore_dump)
-
-            restore_single_dump(
-                repository,
                 config,
                 local_borg_version,
                 global_arguments,
                 local_path,
                 remote_path,
+            )
+            dumps_from_archive = collect_dumps_from_archive(
+                repository['path'],
                 archive_name,
-                restore_dump.hook_name,
+                config,
-                dict(found_data_source, **{'schemas': restore_arguments.schemas}),
+                local_borg_version,
-                connection_params,
+                global_arguments,
+                local_path,
+                remote_path,
                 borgmatic_runtime_directory,
             )
+            dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
 
-        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            dumps_actually_restored = set()
-            'remove_data_source_dumps',
+            connection_params = {
-            config,
+                'container': restore_arguments.container,
-            repository['path'],
+                'hostname': restore_arguments.hostname,
-            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+                'port': restore_arguments.port,
-            borgmatic_runtime_directory,
+                'username': restore_arguments.username,
-            global_arguments.dry_run,
+                'password': restore_arguments.password,
-        )
+                'restore_path': restore_arguments.restore_path,
+            }
+
+            # Restore each dump.
+            for restore_dump in dumps_to_restore:
+                found_data_source = get_configured_data_source(
+                    config,
+                    restore_dump,
+                )
+                # For a dump that wasn't found via an exact match in the configuration, try to fallback
+                # to an "all" data source.
+                if not found_data_source:
+                    found_data_source = get_configured_data_source(
+                        config,
+                        Dump(
+                            restore_dump.hook_name,
+                            'all',
+                            restore_dump.hostname,
+                            restore_dump.port,
+                            restore_dump.label,
+                            restore_dump.container,
+                        ),
+                    )
+
+                    if not found_data_source:
+                        continue
+
+                    found_data_source = dict(found_data_source)
+                    found_data_source['name'] = restore_dump.data_source_name
+
+                dumps_actually_restored.add(restore_dump)
+
+                restore_single_dump(
+                    repository,
+                    config,
+                    local_borg_version,
+                    global_arguments,
+                    local_path,
+                    remote_path,
+                    archive_name,
+                    restore_dump.hook_name,
+                    dict(found_data_source, schemas=restore_arguments.schemas),
+                    connection_params,
+                    borgmatic_runtime_directory,
+                )
 
     ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored)

borgmatic/actions/transfer.py

@@ -17,9 +17,13 @@ def run_transfer(
     '''
     Run the "transfer" action for the given repository.
     '''
-    logger.info(
+    if transfer_arguments.archive and config.get('match_archives'):
-        f'{repository.get("label", repository["path"])}: Transferring archives to repository'
+        raise ValueError(
-    )
+            'With the transfer action, only one of --archive and --match-archives/match_archives can be used.',
+        )
+
+    logger.info('Transferring archives to repository')
+
     borgmatic.borg.transfer.transfer_archives(
         global_arguments.dry_run,
         repository['path'],

borgmatic/borg/borg.py

@@ -29,7 +29,6 @@ def run_arbitrary_borg(
     command.
     '''
     borgmatic.logger.add_custom_log_levels()
-    lock_wait = config.get('lock_wait', None)
 
     try:
         options = options[1:] if options[0] == '--' else options
@@ -39,9 +38,9 @@ def run_arbitrary_borg(
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
 
-        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
+        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES:
             logger.warning(
-                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
+                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}",
             )
     except IndexError:
         borg_command = ()
@@ -53,20 +52,18 @@ def run_arbitrary_borg(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('lock-wait', lock_wait)
+        + flags.make_flags('lock-wait', config.get('lock_wait'))
         + command_options
     )
 
-    return execute_command(
+    return execute_command(  # noqa: S604
         tuple(shlex.quote(part) for part in full_command),
         output_file=DO_NOT_CAPTURE,
         shell=True,
-        extra_environment=dict(
+        environment=dict(
             (environment.make_environment(config) or {}),
-            **{
+            BORG_REPO=repository_path,
-                'BORG_REPO': repository_path,
+            ARCHIVE=archive if archive else '',
-                'ARCHIVE': archive if archive else '',
-            },
         ),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,

borgmatic/borg/break_lock.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, flags
@@ -20,24 +21,25 @@ def break_lock(
     argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
     repository and cache locks leftover from Borg aborting.
     '''
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
+    extra_borg_options = config.get('extra_borg_options', {}).get('break_lock', '')
 
     full_command = (
         (local_path, 'break-lock')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
+        + ('--log-json',)
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    borg_environment = environment.make_environment(config)
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/change_passphrase.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.execute
@@ -23,17 +24,18 @@ def change_passphrase(
     based on an interactive prompt.
     '''
     borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
+    extra_borg_options = config.get('extra_borg_options', {}).get('key_change_passphrase', '')
 
     full_command = (
         (local_path, 'key', 'change-passphrase')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(
             repository_path,
             local_borg_version,
@@ -41,7 +43,7 @@ def change_passphrase(
     )
 
     if global_arguments.dry_run:
-        logger.info(f'{repository_path}: Skipping change password (dry run)')
+        logger.info('Skipping change password (dry run)')
         return
 
     # If the original passphrase is set programmatically, then Borg won't prompt for a new one! So
@@ -49,19 +51,19 @@ def change_passphrase(
     config_without_passphrase = {
         option_name: value
         for (option_name, value) in config.items()
-        if option_name not in ('encryption_passphrase', 'encryption_passcommand')
+        if option_name not in {'encryption_passphrase', 'encryption_passcommand'}
     }
 
     borgmatic.execute.execute_command(
         full_command,
         output_file=borgmatic.execute.DO_NOT_CAPTURE,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config_without_passphrase),
+        environment=environment.make_environment(config_without_passphrase),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
 
     logger.answer(
-        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)"
+        "Don't forget to update your encryption_passphrase option (if needed)",
     )

borgmatic/borg/check.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags, repo_info
@@ -19,7 +20,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument
     flag. And if "prefix" is set in configuration and "archives" is in checks, then include a
     "--match-archives" flag.
     '''
-    check_last = config.get('check_last', None)
+    check_last = config.get('check_last')
     prefix = config.get('prefix')
 
     if 'archives' in checks or 'data' in checks:
@@ -32,7 +33,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument
             if prefix
             else (
                 flags.make_match_archives_flags(
-                    check_arguments.match_archives or config.get('match_archives'),
+                    config.get('match_archives'),
                     config.get('archive_name_format'),
                     local_borg_version,
                 )
@@ -41,11 +42,12 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument
 
     if check_last:
         logger.warning(
-            'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
+            'Ignoring check_last option, as "archives" or "data" are not in consistency checks',
         )
+
     if prefix:
         logger.warning(
-            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
+            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks',
         )
 
     return ()
@@ -64,28 +66,29 @@ def make_check_name_flags(checks, archive_filter_flags):
 
         ('--repository-only',)
 
-    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    However, if both "repository" and "archives" are in checks, then omit the "only" flags from the
-    flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
+    returned flags because Borg does both checks by default. Note that a "data" check only works
+    along with an "archives" check.
     '''
-    if 'data' in checks:
+    data_flags = ('--verify-data',) if 'data' in checks else ()
-        data_flags = ('--verify-data',)
-        checks.update({'archives'})
-    else:
-        data_flags = ()
-
     common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
 
     if {'repository', 'archives'}.issubset(checks):
         return common_flags
 
     return (
-        tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives'))
+        tuple(f'--{check}-only' for check in checks if check in {'repository', 'archives'})
         + common_flags
     )
 
 
 def get_repository_id(
-    repository_path, config, local_borg_version, global_arguments, local_path, remote_path
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Given a local or remote repository path, a configuration dict, the local Borg version, global
@@ -103,7 +106,7 @@ def get_repository_id(
                 global_arguments,
                 local_path,
                 remote_path,
-            )
+            ),
         )['repository']['id']
     except (json.JSONDecodeError, KeyError):
         raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
@@ -130,8 +133,10 @@ def check_archives(
     extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
 
     verbosity_flags = ()
+
     if logger.isEnabledFor(logging.INFO):
         verbosity_flags = ('--info',)
+
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
 
@@ -142,51 +147,59 @@ def check_archives(
     except StopIteration:
         repository_check_config = {}
 
-    if check_arguments.max_duration and 'archives' in checks:
-        raise ValueError('The archives check cannot run when the --max-duration flag is used')
-    if repository_check_config.get('max_duration') and 'archives' in checks:
-        raise ValueError(
-            'The archives check cannot run when the repository check has the max_duration option set'
-        )
-
     max_duration = check_arguments.max_duration or repository_check_config.get('max_duration')
+
+    # If not configured, elevate Borg's exit code 1 (an ostensible warning) to error, because Borg
+    # returns exit code 1 for repository check errors!
+    borg_exit_codes = [*config.get('borg_exit_codes', []), *[{'code': 1, 'treat_as': 'error'}]]
     umask = config.get('umask')
-
-    borg_environment = environment.make_environment(config)
-    borg_exit_codes = config.get('borg_exit_codes')
-
-    full_command = (
-        (local_path, 'check')
-        + (('--repair',) if check_arguments.repair else ())
-        + (('--max-duration', str(max_duration)) if max_duration else ())
-        + make_check_name_flags(checks, archive_filter_flags)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + verbosity_flags
-        + (('--progress',) if check_arguments.progress else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_flags(repository_path, local_borg_version)
-    )
-
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    # The Borg repair option triggers an interactive prompt, which won't work when output is
+    if 'data' in checks:
-    # captured. And progress messes with the terminal directly.
+        checks.add('archives')
-    if check_arguments.repair or check_arguments.progress:
+
+    grouped_checks = (checks,)
+
+    # If max_duration is set, then archives and repository checks need to be run separately, as Borg
+    # doesn't support --max-duration along with an archives checks.
+    if max_duration and 'archives' in checks and 'repository' in checks:
+        checks.remove('repository')
+        grouped_checks = (checks, {'repository'})
+
+    for checks_subset in grouped_checks:
+        full_command = (
+            (local_path, 'check')
+            + (('--repair',) if check_arguments.repair else ())
+            + (
+                ('--max-duration', str(max_duration))
+                if max_duration and 'repository' in checks_subset
+                else ()
+            )
+            + make_check_name_flags(checks_subset, archive_filter_flags)
+            + (('--remote-path', remote_path) if remote_path else ())
+            + (('--umask', str(umask)) if umask else ())
+            + (
+                ('--log-json',)
+                if (
+                    config.get('log_json') or not (check_arguments.repair or config.get('progress'))
+                )
+                else ()
+            )
+            + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+            + verbosity_flags
+            + (('--progress',) if config.get('progress') else ())
+            + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+            + flags.make_repository_flags(repository_path, local_borg_version)
+        )
+
         execute_command(
             full_command,
-            output_file=DO_NOT_CAPTURE,
+            # The Borg repair option triggers an interactive prompt, which won't work when output is
-            extra_environment=borg_environment,
+            # captured. And progress messes with the terminal directly.
-            working_directory=working_directory,
+            output_file=(
-            borg_local_path=local_path,
+                DO_NOT_CAPTURE if check_arguments.repair or config.get('progress') else None
-            borg_exit_codes=borg_exit_codes,
+            ),
-        )
+            environment=environment.make_environment(config),
-    else:
-        execute_command(
-            full_command,
-            extra_environment=borg_environment,
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/compact.py

@@ -1,8 +1,9 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
-from borgmatic.borg import environment, flags
+from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -15,41 +16,46 @@ def compact_segments(
     global_arguments,
     local_path='borg',
     remote_path=None,
-    progress=False,
     cleanup_commits=False,
-    threshold=None,
 ):
     '''
     Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg
     version, compact the segments in a repository.
     '''
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
     extra_borg_options = config.get('extra_borg_options', {}).get('compact', '')
+    threshold = config.get('compact_threshold')
 
     full_command = (
         (local_path, 'compact')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--progress',) if progress else ())
+        + (('--progress',) if config.get('progress') else ())
         + (('--cleanup-commits',) if cleanup_commits else ())
-        + (('--threshold', str(threshold)) if threshold else ())
+        + (('--threshold', str(threshold)) if threshold is not None else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (
+            ('--dry-run',)
+            if dry_run and feature.available(feature.Feature.DRY_RUN_COMPACT, local_borg_version)
+            else ()
+        )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    if dry_run:
+    if dry_run and not feature.available(feature.Feature.DRY_RUN_COMPACT, local_borg_version):
-        logging.info(f'{repository_path}: Skipping compact (dry run)')
+        logger.info('Skipping compact (dry run)')
         return
 
     execute_command(
         full_command,
         output_log_level=logging.INFO,
-        extra_environment=environment.make_environment(config),
+        output_file=DO_NOT_CAPTURE if config.get('progress') else None,
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/create.py

@@ -1,9 +1,8 @@
-import itertools
 import logging
 import os
 import pathlib
+import shlex
 import stat
-import tempfile
 import textwrap
 
 import borgmatic.borg.pattern
@@ -20,76 +19,6 @@ from borgmatic.execute import (
 logger = logging.getLogger(__name__)
 
 
-def write_patterns_file(patterns, borgmatic_runtime_directory, log_prefix, patterns_file=None):
-    '''
-    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
-    temporary file in the given borgmatic runtime directory and return the file object so it can
-    continue to exist on disk as long as the caller needs it.
-
-    Use the given log prefix in any logging.
-
-    If an optional open pattern file is given, append to it instead of making a new temporary file.
-    Return None if no patterns are provided.
-    '''
-    if not patterns:
-        return None
-
-    if patterns_file is None:
-        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
-    else:
-        patterns_file.write('\n')
-
-    patterns_output = '\n'.join(
-        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
-        for pattern in patterns
-    )
-    logger.debug(f'{log_prefix}: Writing patterns to {patterns_file.name}:\n{patterns_output}')
-
-    patterns_file.write(patterns_output)
-    patterns_file.flush()
-
-    return patterns_file
-
-
-def make_exclude_flags(config):
-    '''
-    Given a configuration dict with various exclude options, return the corresponding Borg flags as
-    a tuple.
-    '''
-    caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
-    if_present_flags = tuple(
-        itertools.chain.from_iterable(
-            ('--exclude-if-present', if_present)
-            for if_present in config.get('exclude_if_present', ())
-        )
-    )
-    keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
-    exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
-
-    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
-
-
-def make_list_filter_flags(local_borg_version, dry_run):
-    '''
-    Given the local Borg version and whether this is a dry run, return the corresponding flags for
-    passing to "--list --filter". The general idea is that excludes are shown for a dry run or when
-    the verbosity is debug.
-    '''
-    base_flags = 'AME'
-    show_excludes = logger.isEnabledFor(logging.DEBUG)
-
-    if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
-        if show_excludes or dry_run:
-            return f'{base_flags}+-'
-        else:
-            return base_flags
-
-    if show_excludes:
-        return f'{base_flags}x-'
-    else:
-        return f'{base_flags}-'
-
-
 def special_file(path, working_directory=None):
     '''
     Return whether the given path is a special file (character device, block device, or named pipe
@@ -116,83 +45,117 @@ def any_parent_directories(path, candidate_parents):
     return False
 
 
-def collect_special_file_paths(
+def validate_planned_backup_paths(
     dry_run,
     create_command,
     config,
+    patterns,
     local_path,
     working_directory,
-    borg_environment,
     borgmatic_runtime_directory,
+    find_special_files=False,
 ):
     '''
     Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
-    a working directory, a dict of environment variables to pass to Borg, and the borgmatic runtime
+    a working directory, and the borgmatic runtime directory, perform a "borg create --dry-run" to
-    directory, collect the paths for any special files (character devices, block devices, and named
+    determine whether Borg's planned paths to include in a backup look good. Specifically, if the
-    pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause
+    given runtime directory exists, validate that it will be included in a backup and hasn't been
-    Borg to hang if its --read-special flag is used.
+    excluded.
 
-    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
+    If find special files is True, then return the subset of planned backup paths that are special
-    its own special files there for database dumps. And if the borgmatic runtime directory is
+    files. Otherwise, return an empty tuple.
-    configured to be excluded from the files Borg backs up, error, because this means Borg won't be
+
-    able to consume any database dumps and therefore borgmatic will hang.
+    Raise ValueError if the runtime directory has been excluded via "exclude_patterns" or similar,
+    because any features that rely on the runtime directory getting backed up will break. For
+    instance, without the runtime directory, Borg can't consume any database dumps and borgmatic may
+    hang waiting for them to be consumed.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
-    # files including any named pipe we've created.
+    # files including any named pipe we've created. And omit "--filter" because that can break the
-    paths_output = execute_command_and_capture_output(
+    # paths output parsing below such that path lines no longer start with the expected "- ".
-        tuple(argument for argument in create_command if argument != '--exclude-nodump')
+    path_lines = execute_command_and_capture_output(
-        + ('--dry-run', '--list'),
+        (
+            *flags.omit_flag_and_value(
+                flags.omit_flag(
+                    flags.omit_flag(create_command, '--exclude-nodump'),
+                    '--log-json',
+                ),
+                '--filter',
+            ),
+            '--dry-run',
+            '--list',
+        ),
         capture_stderr=True,
         working_directory=working_directory,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
 
-    paths = tuple(
+    # These are all the individual files that Borg is planning to backup as determined by the Borg
+    # create dry run above.
+    paths = (
         path_line.split(' ', 1)[1]
-        for path_line in paths_output.split('\n')
+        for path_line in path_lines
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
+        if path_line and path_line.startswith(('- ', '+ '))
-    )
-    skip_paths = {}
-
-    if os.path.exists(borgmatic_runtime_directory):
-        skip_paths = {
-            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
-        }
-
-        if not skip_paths and not dry_run:
-            raise ValueError(
-                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
-            )
-
-    return tuple(
-        path for path in paths if special_file(path, working_directory) if path not in skip_paths
     )
 
-
+    include_pattern_paths = {
-def check_all_root_patterns_exist(patterns):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
-    paths exist. If any don't, raise an exception.
-    '''
-    missing_paths = [
         pattern.path
         for pattern in patterns
+        if pattern.type == borgmatic.borg.pattern.Pattern_type.INCLUDE
+    }
+    runtime_directory_root_patterns = tuple(
+        pattern
+        for pattern in patterns
+        if any_parent_directories(pattern.path, (borgmatic_runtime_directory,))
         if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-        if not os.path.exists(pattern.path)
+        # Skip root patterns that have corresponding include patterns, because those will "punch
-    ]
+        # through" any subsequent excludes.
+        if pattern.path not in include_pattern_paths
+    )
 
-    if missing_paths:
+    special_paths = []
-        raise ValueError(
+    validate_runtime_directory = bool(
-            f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}"
+        not dry_run
+        and os.path.exists(borgmatic_runtime_directory)
+        and runtime_directory_root_patterns
+    )
+    runtime_directory_in_path = False
+
+    # Do everything in this one loop because we only want to consume the paths generator once.
+    for path in paths:
+        # If all root patterns in the runtime directory are missing from the paths Borg is planning
+        # to backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn
+        # accordingly (below).
+        if validate_runtime_directory and any(
+            any_parent_directories(path, (pattern.path,))
+            for pattern in runtime_directory_root_patterns
+        ):
+            runtime_directory_in_path = True
+
+        # Return the subset of output paths that are special files but *not* contained within the
+        # borgmatic runtime directory. The intent is to skip runtime paths that borgmatic uses for its
+        # own bookkeeping, instead focusing on user-configured paths.
+        if (
+            find_special_files
+            and not any_parent_directories(path, (borgmatic_runtime_directory,))
+            and special_file(path, working_directory)
+        ):
+            special_paths.append(path)
+
+    if validate_runtime_directory and not runtime_directory_in_path:
+        logger.warning(
+            f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes (or the snapshotted source directories are empty). Please ensure the runtime directory is not excluded.'
         )
 
+    return tuple(special_paths)
+
 
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
 
 
-def make_base_create_command(
+def make_base_create_command(  # noqa: PLR0912
     dry_run,
     repository_path,
     config,
@@ -200,38 +163,45 @@ def make_base_create_command(
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
+    archive_suffix='',
     local_path='borg',
     remote_path=None,
-    progress=False,
     json=False,
-    list_files=False,
+    comment=None,
     stream_processes=None,
 ):
     '''
     Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version,
+    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version, global
-    global arguments as an argparse.Namespace instance, and a sequence of borgmatic source
+    arguments as an argparse.Namespace instance, the borgmatic runtime directory, a string suffix to
-    directories, return a tuple of (base Borg create command flags, Borg create command positional
+    add to the archive name, the local Borg path, the remote Borg path, whether to output JSON,
-    arguments, open pattern file handle).
+    comment text to add to the created archive, and a sequence of processes streaming data to Borg,
+    return a tuple of (base Borg create command flags, Borg create command positional arguments,
+    open pattern file handle).
     '''
-    if config.get('source_directories_must_exist', False):
+    if config.get('source_directories_must_exist', True):
-        check_all_root_patterns_exist(patterns)
+        borgmatic.borg.pattern.check_all_root_patterns_exist(
+            patterns, borgmatic.config.paths.get_working_directory(config)
+        )
 
-    patterns_file = write_patterns_file(
+    patterns_file = borgmatic.borg.pattern.write_patterns_file(
-        patterns, borgmatic_runtime_directory, log_prefix=repository_path
+        patterns,
+        borgmatic_runtime_directory,
     )
-    checkpoint_interval = config.get('checkpoint_interval', None)
+    checkpoint_interval = config.get('checkpoint_interval')
-    checkpoint_volume = config.get('checkpoint_volume', None)
+    checkpoint_volume = config.get('checkpoint_volume')
-    chunker_params = config.get('chunker_params', None)
+    chunker_params = config.get('chunker_params')
-    compression = config.get('compression', None)
+    compression = config.get('compression')
-    upload_rate_limit = config.get('upload_rate_limit', None)
+    upload_rate_limit = config.get('upload_rate_limit')
-    upload_buffer_size = config.get('upload_buffer_size', None)
+    upload_buffer_size = config.get('upload_buffer_size')
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
-    list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
+    list_filter_flags = flags.make_list_filter_flags(local_borg_version, dry_run)
+    files_changed = config.get('files_changed')
     files_cache = config.get('files_cache')
-    archive_name_format = config.get(
+    archive_name_format = (
-        'archive_name_format', flags.get_default_archive_name_format(local_borg_version)
+        config.get('archive_name_format', flags.get_default_archive_name_format(local_borg_version))
+        + archive_suffix
     )
     extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
 
@@ -263,7 +233,8 @@ def make_base_create_command(
         tuple(local_path.split(' '))
         + ('create',)
         + (('--patterns-from', patterns_file.name) if patterns_file else ())
-        + make_exclude_flags(config)
+        + flags.make_exclude_flags(config)
+        + (('--comment', comment) if comment else ())
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
@@ -277,69 +248,80 @@ def make_base_create_command(
         + (('--nobirthtime',) if config.get('birthtime') is False else ())
         + (('--read-special',) if config.get('read_special') or stream_processes else ())
         + noflags_flags
+        + (('--files-changed', files_changed) if files_changed else ())
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
         + (
             ('--list', '--filter', list_filter_flags)
-            if list_files and not json and not progress
+            if config.get('list_details') and not json and not config.get('progress')
             else ()
         )
         + (('--dry-run',) if dry_run else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
     )
 
     create_positional_arguments = flags.make_repository_archive_flags(
-        repository_path, archive_name_format, local_borg_version
+        repository_path,
+        archive_name_format,
+        local_borg_version,
     )
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    if config.get('unsafe_skip_path_validation_before_create'):
+        logger.debug(
+            'Skipping pre-backup path validation due to "unsafe_skip_path_validation_before_create" option.'
+        )
+
+        return (create_flags, create_positional_arguments, patterns_file)
+
+    logger.debug('Checking file paths Borg plans to include')
+
+    special_file_paths = validate_planned_backup_paths(
+        dry_run,
+        create_flags + create_positional_arguments,
+        config,
+        patterns,
+        local_path,
+        working_directory,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
+        find_special_files=bool(stream_processes),
+    )
+
+    if stream_processes and config.get('read_special') is False:
+        logger.warning(
+            'Ignoring configured "read_special" value of false, as true is needed for database hooks.',
+        )
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
-    if stream_processes and not config.get('read_special'):
+    if special_file_paths:
+        truncated_special_file_paths = textwrap.shorten(
+            ', '.join(special_file_paths),
+            width=MAX_SPECIAL_FILE_PATHS_LENGTH,
+            placeholder=' ...',
+        )
         logger.warning(
-            f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
+            f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
         )
-        borg_environment = environment.make_environment(config)
+        patterns_file = borgmatic.borg.pattern.write_patterns_file(
-        working_directory = borgmatic.config.paths.get_working_directory(config)
+            tuple(
-
+                borgmatic.borg.pattern.Pattern(
-        logger.debug(f'{repository_path}: Collecting special file paths')
+                    special_file_path,
-        special_file_paths = collect_special_file_paths(
+                    borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
-            dry_run,
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-            create_flags + create_positional_arguments,
+                    source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
-            config,
+                )
-            local_path,
+                for special_file_path in special_file_paths
-            working_directory,
+            ),
-            borg_environment,
+            borgmatic_runtime_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
+            patterns_file=patterns_file,
         )
 
-        if special_file_paths:
+        if '--patterns-from' not in create_flags:
-            truncated_special_file_paths = textwrap.shorten(
+            create_flags += ('--patterns-from', patterns_file.name)
-                ', '.join(special_file_paths),
-                width=MAX_SPECIAL_FILE_PATHS_LENGTH,
-                placeholder=' ...',
-            )
-            logger.warning(
-                f'{repository_path}: Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
-            )
-            patterns_file = write_patterns_file(
-                tuple(
-                    borgmatic.borg.pattern.Pattern(
-                        special_file_path,
-                        borgmatic.borg.pattern.Pattern_type.EXCLUDE,
-                        borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                    )
-                    for special_file_path in special_file_paths
-                ),
-                borgmatic_runtime_directory,
-                log_prefix=repository_path,
-                patterns_file=patterns_file,
-            )
-
-            if '--patterns-from' not in create_flags:
-                create_flags += ('--patterns-from', patterns_file.name)
 
     return (create_flags, create_positional_arguments, patterns_file)
 
@@ -352,18 +334,20 @@ def create_archive(
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
+    archive_suffix='',
     local_path='borg',
     remote_path=None,
-    progress=False,
-    stats=False,
     json=False,
-    list_files=False,
+    comment=None,
     stream_processes=None,
 ):
     '''
     Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of loaded configuration paths, the local Borg version, and global arguments as an
+    sequence of loaded configuration paths, the local Borg version, global arguments as an
-    argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
+    argparse.Namespace instance, the borgmatic runtime directory, a string suffix to add to the
+    archive name, the local Borg path, the remote Borg path, whether to output JSON, and comment
+    text to add to the created archive, and a sequence of processes streaming data to Borg, create a
+    Borg archive and return Borg's JSON output (if any).
 
     If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
     create command while also triggering the given processes to produce output.
@@ -372,7 +356,7 @@ def create_archive(
 
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
+    (create_flags, create_positional_arguments, _) = make_base_create_command(
         dry_run,
         repository_path,
         config,
@@ -380,62 +364,76 @@ def create_archive(
         local_borg_version,
         global_arguments,
         borgmatic_runtime_directory,
+        archive_suffix,
         local_path,
         remote_path,
-        progress,
         json,
-        list_files,
+        comment,
         stream_processes,
     )
 
     if json:
         output_log_level = None
-    elif list_files or (stats and not dry_run):
+    elif config.get('list_details') or (
+        (config.get('statistics') or config.get('quick_statistics')) and not dry_run
+    ):
         output_log_level = logging.ANSWER
     else:
         output_log_level = logging.INFO
 
     # The progress output isn't compatible with captured and logged output, as progress messes with
     # the terminal directly.
-    output_file = DO_NOT_CAPTURE if progress else None
+    output_file = DO_NOT_CAPTURE if config.get('progress') else None
-
-    borg_environment = environment.make_environment(config)
 
     create_flags += (
         (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
-        + (('--stats',) if stats and not json and not dry_run else ())
+        + (('--stats',) if config.get('statistics') and not json and not dry_run else ())
+        + (
+            ('--quick-stats',)
+            if config.get('quick_statistics') and not json and not dry_run
+            else ()
+        )
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
-        + (('--progress',) if progress else ())
+        + (('--progress',) if config.get('progress') else ())
         + (('--json',) if json else ())
     )
     borg_exit_codes = config.get('borg_exit_codes')
 
     if stream_processes:
-        return execute_command_with_processes(
+        output = '\n'.join(
-            create_flags + create_positional_arguments,
+            execute_command_with_processes(
-            stream_processes,
+                create_flags + create_positional_arguments,
-            output_log_level,
+                stream_processes,
-            output_file,
+                output_log_level,
-            working_directory=working_directory,
+                output_file,
-            extra_environment=borg_environment,
+                working_directory=working_directory,
-            borg_local_path=local_path,
+                environment=environment.make_environment(config),
-            borg_exit_codes=borg_exit_codes,
+                borg_local_path=local_path,
+                borg_exit_codes=borg_exit_codes,
+            )
         )
-    elif output_log_level is None:
+        return output if json else None
-        return execute_command_and_capture_output(
+
-            create_flags + create_positional_arguments,
+    if output_log_level is None:
-            working_directory=working_directory,
+        output = '\n'.join(
-            extra_environment=borg_environment,
+            execute_command_and_capture_output(
-            borg_local_path=local_path,
+                create_flags + create_positional_arguments,
-            borg_exit_codes=borg_exit_codes,
+                working_directory=working_directory,
-        )
+                environment=environment.make_environment(config),
-    else:
+                borg_local_path=local_path,
-        execute_command(
+                borg_exit_codes=borg_exit_codes,
-            create_flags + create_positional_arguments,
+            )
-            output_log_level,
-            output_file,
-            working_directory=working_directory,
-            extra_environment=borg_environment,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
         )
+        return output if json else None
+
+    execute_command(
+        create_flags + create_positional_arguments,
+        output_log_level,
+        output_file,
+        working_directory=working_directory,
+        environment=environment.make_environment(config),
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )
+
+    return None

borgmatic/borg/delete.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import shlex
 
 import borgmatic.borg.environment
 import borgmatic.borg.feature
@@ -11,6 +12,9 @@ import borgmatic.execute
 logger = logging.getLogger(__name__)
 
 
+FORCE_HARDER_FLAG_COUNT = 2
+
+
 def make_delete_command(
     repository,
     config,
@@ -25,6 +29,8 @@ def make_delete_command(
     arguments to the delete action as an argparse.Namespace, and global arguments, return a command
     as a tuple to delete archives from the repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('delete', '')
+
     return (
         (local_path, 'delete')
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
@@ -32,11 +38,14 @@ def make_delete_command(
         + borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
         + borgmatic.borg.flags.make_flags('remote-path', remote_path)
         + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
-        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+        + ('--log-json',)
         + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
-        + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives)
+        + borgmatic.borg.flags.make_flags('list', config.get('list_details'))
         + (
-            (('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
+            (
+                ('--force',)
+                + (('--force',) if delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
+            )
             if delete_arguments.force
             else ()
         )
@@ -48,10 +57,19 @@ def make_delete_command(
             local_borg_version=local_borg_version,
             default_archive_name_format='*',
         )
+        + (('--stats',) if config.get('statistics') else ())
         + borgmatic.borg.flags.make_flags_from_arguments(
             delete_arguments,
-            excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'),
+            excludes=(
+                'list_details',
+                'statistics',
+                'force',
+                'match_archives',
+                'archive',
+                'repository',
+            ),
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
     )
 
@@ -90,15 +108,16 @@ def delete_archives(
         for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
     ):
         if borgmatic.borg.feature.available(
-            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+            borgmatic.borg.feature.Feature.REPO_DELETE,
+            local_borg_version,
         ):
             logger.warning(
-                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.'
+                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.',
             )
 
         repo_delete_arguments = argparse.Namespace(
             repository=repository['path'],
-            list_archives=delete_arguments.list_archives,
+            list_details=delete_arguments.list_details,
             force=delete_arguments.force,
             cache_only=delete_arguments.cache_only,
             keep_security_info=delete_arguments.keep_security_info,
@@ -128,7 +147,7 @@ def delete_archives(
     borgmatic.execute.execute_command(
         command,
         output_log_level=logging.ANSWER,
-        extra_environment=borgmatic.borg.environment.make_environment(config),
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/diff.py

@@ -0,0 +1,88 @@
+import logging
+import shlex
+
+import borgmatic.borg.environment
+import borgmatic.borg.feature
+import borgmatic.config.paths
+import borgmatic.execute
+from borgmatic.borg import flags
+from borgmatic.borg.pattern import write_patterns_file
+
+logger = logging.getLogger(__name__)
+
+
+def diff(
+    repository,
+    archive,
+    second_archive,
+    config,
+    local_borg_version,
+    diff_arguments,
+    global_arguments,
+    local_path,
+    remote_path=None,
+    patterns=None,
+):
+    '''
+    Given a local or remote repository path, two archive names, a configuration dict, the local Borg
+    version string, an argparse.Namespace of diff arguments, an argparse.Namespace of global
+    arguments, optional local and remote Borg paths, executes the diff command with the given
+    arguments.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    lock_wait = config.get('lock_wait')
+    extra_borg_options = config.get('extra_borg_options', {}).get('diff', '')
+
+    if diff_arguments.only_patterns:
+        # Write patterns to a temporary file and use that file with --patterns-from.
+        patterns_file = write_patterns_file(
+            patterns,
+            borgmatic.config.paths.get_working_directory(config),
+        )
+    else:
+        patterns_file = None
+
+    if borgmatic.borg.feature.available(
+        borgmatic.borg.feature.Feature.NUMERIC_IDS, local_borg_version
+    ):
+        numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
+    else:
+        numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
+
+    diff_command = (
+        (local_path, 'diff')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + ('--log-json',)
+        + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (
+            ('--patterns-from', patterns_file.name)
+            if patterns_file and diff_arguments.only_patterns
+            else ()
+        )
+        + numeric_ids_flags
+        + (('--same-chunker-params',) if diff_arguments.same_chunker_params else ())
+        + (('--sort-by', ','.join(diff_arguments.sort_keys)) if diff_arguments.sort_keys else ())
+        + (('--content-only',) if diff_arguments.content_only else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + (
+            (*flags.make_repository_flags(repository, local_borg_version), archive)
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE,
+                local_borg_version,
+            )
+            else flags.make_repository_archive_flags(repository, archive, local_borg_version)
+        )
+        + (second_archive,)
+    )
+
+    borgmatic.execute.execute_command(
+        full_command=diff_command,
+        output_log_level=logging.ANSWER,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/environment.py

@@ -1,53 +1,105 @@
 import os
 
+import borgmatic.borg.passcommand
+import borgmatic.hooks.credential.parse
+
 OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'archive_hostname': 'BORG_HOSTNAME',
+    'archive_username': 'BORG_USERNAME',
     'borg_base_directory': 'BORG_BASE_DIR',
     'borg_config_directory': 'BORG_CONFIG_DIR',
     'borg_cache_directory': 'BORG_CACHE_DIR',
     'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
-    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'borg_key_file': 'BORG_KEY_FILE',
-    'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
 }
 
-DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = {
+DEFAULT_BOOL_OPTION_TO_UNCONDITIONAL_ENVIRONMENT_VARIABLE = {
-    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
-    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
 }
 
-DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
-    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
+    'debug_passphrase': 'BORG_DEBUG_PASSPHRASE',
+    'display_passphrase': 'BORG_DISPLAY_PASSPHRASE',
+    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
+    'use_chunks_archive': 'BORG_USE_CHUNKS_ARCHIVE',
+    'msgpack_version_check': 'BORG_MSGPACK_VERSION_CHECK',
 }
 
 
 def make_environment(config):
     '''
-    Given a borgmatic configuration dict, return its options converted to a Borg environment
+    Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it
-    variable dict.
+    with a copy of the current environment variables, and return the result.
+
+    Do not reuse this environment across multiple Borg invocations, because it can include
+    references to resources like anonymous pipes for passphrases—which can only be consumed once.
+
+    Here's how native Borg precedence works for a few of the environment variables:
+
+      1. BORG_PASSPHRASE, if set, is used first.
+      2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set.
+      3. BORG_PASSPHRASE_FD is used only if neither of the above are set.
+
+    In borgmatic, we want to simulate this precedence order, but there are some additional
+    complications. First, values can come from either configuration or from environment variables
+    set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a
+    passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we
+    want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's
+    more secure than using an environment variable (BORG_PASSPHRASE).
     '''
-    environment = {}
+    environment = dict(os.environ)
 
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
 
-        if value:
+        if value is not None:
             environment[environment_variable_name] = str(value)
 
-    for (
+    if 'encryption_passphrase' in config:
-        option_name,
+        environment.pop('BORG_PASSPHRASE', None)
-        environment_variable_name,
+        environment.pop('BORG_PASSCOMMAND', None)
-    ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
+
-        if os.environ.get(environment_variable_name) is None:
+    if 'encryption_passcommand' in config:
-            value = config.get(option_name)
+        environment.pop('BORG_PASSCOMMAND', None)
-            environment[environment_variable_name] = 'yes' if value else 'no'
+
+    passphrase = borgmatic.hooks.credential.parse.resolve_credential(
+        config.get('encryption_passphrase'),
+        config,
+    )
+
+    if passphrase is None:
+        passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
+
+    # If there's a passphrase (from configuration, from a configured credential, or from a
+    # configured passcommand), send it to Borg via an anonymous pipe.
+    if passphrase is not None:
+        read_file_descriptor, write_file_descriptor = os.pipe()
+        os.write(write_file_descriptor, passphrase.encode('utf-8'))
+        os.close(write_file_descriptor)
+
+        # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg
+        # child process to inherit the file descriptor.
+        os.set_inheritable(read_file_descriptor, True)
+        environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor)
 
     for (
         option_name,
         environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items():
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        if os.environ.get(environment_variable_name) is None:
+            value = config.get(option_name)
+            if value is not None:
+                environment[environment_variable_name] = 'YES' if value else 'NO'
+
+    for (
+        option_name,
+        environment_variable_name,
+    ) in DEFAULT_BOOL_OPTION_TO_UNCONDITIONAL_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
         if value is not None:
             environment[environment_variable_name] = 'YES' if value else 'NO'

borgmatic/borg/export_key.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -28,14 +29,15 @@ def export_key(
     Raise FileExistsError if a path is given but it already exists on disk.
     '''
     borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
     working_directory = borgmatic.config.paths.get_working_directory(config)
+    extra_borg_options = config.get('extra_borg_options', {}).get('key_export', '')
 
     if export_arguments.path and export_arguments.path != '-':
         if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
             raise FileExistsError(
-                f'Destination path {export_arguments.path} already exists. Aborting.'
+                f'Destination path {export_arguments.path} already exists. Aborting.',
             )
 
         output_file = None
@@ -46,12 +48,13 @@ def export_key(
         (local_path, 'key', 'export')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--log-json',) if output_file is None else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('paper', export_arguments.paper)
         + flags.make_flags('qr-html', export_arguments.qr_html)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(
             repository_path,
             local_borg_version,
@@ -60,14 +63,14 @@ def export_key(
     )
 
     if global_arguments.dry_run:
-        logger.info(f'{repository_path}: Skipping key export (dry run)')
+        logger.info('Skipping key export (dry run)')
         return
 
     execute_command(
         full_command,
         output_file=output_file,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/export_tar.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -20,7 +21,6 @@ def export_tar_archive(
     local_path='borg',
     remote_path=None,
     tar_filter=None,
-    list_files=False,
     strip_components=None,
 ):
     '''
@@ -33,21 +33,23 @@ def export_tar_archive(
     If the destination path is "-", then stream the output to stdout instead of to a file.
     '''
     borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
+    extra_borg_options = config.get('extra_borg_options', {}).get('export_tar', '')
 
     full_command = (
         (local_path, 'export-tar')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--log-json',) if destination_path != '-' else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--list',) if list_files else ())
+        + (('--list',) if config.get('list_details') else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--tar-filter', tar_filter) if tar_filter else ())
         + (('--strip-components', str(strip_components)) if strip_components else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_archive_flags(
             repository_path,
             archive,
@@ -57,20 +59,17 @@ def export_tar_archive(
         + (tuple(paths) if paths else ())
     )
 
-    if list_files:
+    output_log_level = logging.ANSWER if config.get('list_details') else logging.INFO
-        output_log_level = logging.ANSWER
-    else:
-        output_log_level = logging.INFO
 
     if dry_run:
-        logging.info(f'{repository_path}: Skipping export to tar file (dry run)')
+        logger.info('Skipping export to tar file (dry run)')
         return
 
     execute_command(
         full_command,
         output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
         output_log_level=output_log_level,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/extract.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import shlex
 import subprocess
 
 import borgmatic.config.paths
@@ -23,6 +24,7 @@ def extract_last_archive_dry_run(
     Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
     dry-run.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('extract', '')
     verbosity_flags = ()
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
@@ -44,26 +46,39 @@ def extract_last_archive_dry_run(
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
-    borg_environment = environment.make_environment(config)
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
         + (('--remote-path', remote_path) if remote_path else ())
-        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--log-json',) if not config.get('progress') else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + verbosity_flags
+        + (('--progress',) if config.get('progress') else ())
         + list_flag
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_archive_flags(
-            repository_path, last_archive_name, local_borg_version
+            repository_path,
+            last_archive_name,
+            local_borg_version,
         )
     )
 
-    execute_command(
+    if config.get('progress'):
-        full_extract_command,
+        execute_command(
-        extra_environment=borg_environment,
+            full_extract_command,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+            output_file=DO_NOT_CAPTURE,
-        borg_local_path=local_path,
+            environment=environment.make_environment(config),
-        borg_exit_codes=config.get('borg_exit_codes'),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
-    )
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
+        )
+    else:
+        execute_command(
+            full_extract_command,
+            environment=environment.make_environment(config),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
+        )
 
 
 def extract_archive(
@@ -78,7 +93,6 @@ def extract_archive(
     remote_path=None,
     destination_path=None,
     strip_components=None,
-    progress=False,
     extract_to_stdout=False,
 ):
     '''
@@ -90,11 +104,9 @@ def extract_archive(
     If extract to stdout is True, then start the extraction streaming to stdout, and return that
     extract process as an instance of subprocess.Popen.
     '''
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
-
+    extra_borg_options = config.get('extra_borg_options', {}).get('extract', '')
-    if progress and extract_to_stdout:
-        raise ValueError('progress and extract_to_stdout cannot both be set')
 
     if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
         numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
@@ -123,63 +135,63 @@ def extract_archive(
         + (('--remote-path', remote_path) if remote_path else ())
         + numeric_ids_flags
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--strip-components', str(strip_components)) if strip_components else ())
-        + (('--progress',) if progress else ())
+        + (('--progress',) if config.get('progress') and not extract_to_stdout else ())
         + (('--stdout',) if extract_to_stdout else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_archive_flags(
             # Make the repository path absolute so the destination directory used below via changing
             # the working directory doesn't prevent Borg from finding the repo. But also apply the
             # user's configured working directory (if any) to the repo path.
-            borgmatic.config.validate.normalize_repository_path(
+            borgmatic.config.validate.normalize_repository_path(repository, working_directory),
-                os.path.join(working_directory or '', repository)
-            ),
             archive,
             local_borg_version,
         )
         + (tuple(paths) if paths else ())
     )
 
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
     full_destination_path = (
         os.path.join(working_directory or '', destination_path) if destination_path else None
     )
 
-    # The progress output isn't compatible with captured and logged output, as progress messes with
-    # the terminal directly.
-    if progress:
-        return execute_command(
-            full_command,
-            output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
-            working_directory=full_destination_path,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
-        )
-        return None
-
     if extract_to_stdout:
         return execute_command(
             full_command,
             output_file=subprocess.PIPE,
             run_to_completion=False,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=full_destination_path,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
 
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if config.get('progress'):
+        return execute_command(
+            full_command,
+            output_file=DO_NOT_CAPTURE,
+            environment=environment.make_environment(config),
+            working_directory=full_destination_path,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
+        )
+        return None
+
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=full_destination_path,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
+
+    return None

borgmatic/borg/feature.py

@@ -17,6 +17,8 @@ class Feature(Enum):
     MATCH_ARCHIVES = 11
     EXCLUDED_FILES_MINUS = 12
     ARCHIVE_SERIES = 13
+    NO_PRUNE_STATS = 14
+    DRY_RUN_COMPACT = 15
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
@@ -33,6 +35,8 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
     Feature.MATCH_ARCHIVES: parse('2.0.0b3'),  # borg --match-archives
     Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
     Feature.ARCHIVE_SERIES: parse('2.0.0b11'),  # identically named archives form a series
+    Feature.NO_PRUNE_STATS: parse('2.0.0b10'),  # prune --stats is not available
+    Feature.DRY_RUN_COMPACT: parse('1.2.9'),  # borg compact --dry-run support
 }
 
 

borgmatic/borg/flags.py

@@ -1,5 +1,4 @@
 import itertools
-import json
 import logging
 import re
 
@@ -34,7 +33,7 @@ def make_flags_from_arguments(arguments, excludes=()):
             make_flags(name, value=getattr(arguments, name))
             for name in sorted(vars(arguments))
             if name not in excludes and not name.startswith('_')
-        )
+        ),
     )
 
 
@@ -50,7 +49,7 @@ def make_repository_flags(repository_path, local_borg_version):
     ) + (repository_path,)
 
 
-ARCHIVE_HASH_PATTERN = re.compile('[0-9a-fA-F]{8,}$')
+ARCHIVE_HASH_PATTERN = re.compile(r'[0-9a-fA-F]{8,}$')
 
 
 def make_repository_archive_flags(repository_path, archive, local_borg_version):
@@ -76,8 +75,8 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
     )
 
 
-DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
+DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'  # noqa: FS003
+DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'
 
 
 def get_default_archive_name_format(local_borg_version):
@@ -90,7 +89,7 @@ def get_default_archive_name_format(local_borg_version):
     return DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES
 
 
-def make_match_archives_flags(
+def make_match_archives_flags(  # noqa: PLR0911
     match_archives,
     archive_name_format,
     local_borg_version,
@@ -115,8 +114,8 @@ def make_match_archives_flags(
                 return ('--match-archives', f'aid:{match_archives}')
 
             return ('--match-archives', match_archives)
-        else:
+
-            return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
+        return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
 
     derived_match_archives = re.sub(
         r'\{(now|utcnow|pid)([:%\w\.-]*)\}',
@@ -131,28 +130,99 @@ def make_match_archives_flags(
 
     if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
         return ('--match-archives', f'sh:{derived_match_archives}')
-    else:
+
-        return ('--glob-archives', f'{derived_match_archives}')
+    return ('--glob-archives', f'{derived_match_archives}')
 
 
-def warn_for_aggressive_archive_flags(json_command, json_output):
+def warn_for_aggressive_archive_flags(command, output_lines):
     '''
-    Given a JSON archives command and the resulting JSON string output from running it, parse the
+    Given an archives command and the resulting output lines from running it, warn if the command
-    JSON and warn if the command used an archive flag but the output indicates zero archives were
+    used an archive flag but the output indicates zero archives were found.
-    found.
     '''
-    archive_flags_used = {'--glob-archives', '--match-archives'}.intersection(set(json_command))
+    if {'--glob-archives', '--match-archives'}.intersection(set(command)) and len(
+        tuple(line for line in output_lines if not line.startswith('terminating with '))
+    ) == 0:
+        logger.warning('An archive filter was applied, but no matching archives were found.')
+        logger.warning(
+            'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.',
+        )
 
-    if not archive_flags_used:
-        return
 
-    try:
+def omit_flag(arguments, flag):
-        if len(json.loads(json_output)['archives']) == 0:
+    '''
-            logger.warning('An archive filter was applied, but no matching archives were found.')
+    Given a sequence of Borg command-line arguments, return them with the given (valueless) flag
-            logger.warning(
+    omitted. For instance, if the flag is "--flag" and arguments is:
-                'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.'
+
-            )
+        ('borg', 'create', '--flag', '--other-flag')
-    except json.JSONDecodeError as error:
+
-        logger.debug(f'Cannot parse JSON output from archive command: {error}')
+    ... then return:
-    except (TypeError, KeyError):
+
-        logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
+        ('borg', 'create', '--other-flag')
+    '''
+    return tuple(argument for argument in arguments if argument != flag)
+
+
+def omit_flag_and_value(arguments, flag):
+    '''
+    Given a sequence of Borg command-line arguments, return them with the given flag and its
+    corresponding value omitted. For instance, if the flag is "--flag" and arguments is:
+
+        ('borg', 'create', '--flag', 'value', '--other-flag')
+
+    ... or:
+
+        ('borg', 'create', '--flag=value', '--other-flag')
+
+    ... then return:
+
+        ('borg', 'create', '--other-flag')
+    '''
+    # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two',
+    # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')).
+    # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and
+    # its value.
+    return tuple(
+        argument
+        for (previous_argument, argument) in zip((None, *arguments), arguments)
+        if flag not in {previous_argument, argument}
+        if not argument.startswith(f'{flag}=')
+    )
+
+
+def make_exclude_flags(config):
+    '''
+    Given a configuration dict with various exclude options, return the corresponding Borg flags as
+    a tuple.
+    '''
+    caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
+    if_present_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-if-present', if_present)
+            for if_present in config.get('exclude_if_present', ())
+        ),
+    )
+    keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
+    exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
+
+    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
+
+
+def make_list_filter_flags(local_borg_version, dry_run):
+    '''
+    Given the local Borg version and whether this is a dry run, return the corresponding flags for
+    passing to "--list --filter". The general idea is that excludes are shown for a dry run or when
+    the verbosity is debug.
+    '''
+    base_flags = 'AME'
+    show_excludes = logger.isEnabledFor(logging.DEBUG)
+
+    if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
+        if show_excludes or dry_run:
+            return f'{base_flags}+-'
+
+        return base_flags
+
+    if show_excludes:
+        return f'{base_flags}x-'
+
+    return f'{base_flags}-'

borgmatic/borg/import_key.py

@@ -0,0 +1,70 @@
+import logging
+import os
+import shlex
+
+import borgmatic.config.paths
+from borgmatic.borg import environment, flags
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def import_key(
+    repository_path,
+    config,
+    local_borg_version,
+    import_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, import
+    arguments, and optional local and remote Borg paths, import the repository key from the
+    path indicated in the import arguments.
+
+    If the path is empty or "-", then read the key from stdin.
+
+    Raise ValueError if the path is given and it does not exist.
+    '''
+    umask = config.get('umask')
+    lock_wait = config.get('lock_wait')
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+    extra_borg_options = config.get('extra_borg_options', {}).get('key_import', '')
+
+    if (
+        import_arguments.path
+        and import_arguments.path != '-'
+        and not os.path.exists(os.path.join(working_directory or '', import_arguments.path))
+    ):
+        raise ValueError(f'Path {import_arguments.path} does not exist. Aborting.')
+
+    full_command = (
+        (local_path, 'key', 'import')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + ('--log-json',)
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + flags.make_flags('paper', import_arguments.paper)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + flags.make_repository_flags(
+            repository_path,
+            local_borg_version,
+        )
+        + (import_arguments.path or '-',)
+    )
+
+    if global_arguments.dry_run:
+        logger.info('Skipping key import (dry run)')
+        return
+
+    execute_command(
+        full_command,
+        output_log_level=logging.INFO,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/info.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command, execute_command_and_capture_output
+from borgmatic.execute import execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +24,8 @@ def make_info_command(
     arguments to the info action as an argparse.Namespace, and global arguments, return a command
     as a tuple to display summary information for archives in the repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('info', '')
+
     return (
         (local_path, 'info')
         + (
@@ -37,7 +40,7 @@ def make_info_command(
         )
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
+        + ('--log-json',)
         + flags.make_flags('lock-wait', config.get('lock_wait'))
         + (
             (
@@ -48,17 +51,17 @@ def make_info_command(
             if info_arguments.prefix
             else (
                 flags.make_match_archives_flags(
-                    info_arguments.match_archives
+                    info_arguments.archive or config.get('match_archives'),
-                    or info_arguments.archive
-                    or config.get('match_archives'),
                     config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
         )
         + flags.make_flags_from_arguments(
-            info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
+            info_arguments,
+            excludes=('repository', 'archive', 'prefix', 'match_archives'),
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
@@ -100,24 +103,28 @@ def display_archives_info(
     borg_exit_codes = config.get('borg_exit_codes')
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    json_info = execute_command_and_capture_output(
-        json_command,
-        extra_environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=borg_exit_codes,
-    )
-
     if info_arguments.json:
-        return json_info
+        return '\n'.join(
+            execute_command_and_capture_output(
+                json_command,
+                environment=environment.make_environment(config),
+                working_directory=working_directory,
+                borg_local_path=local_path,
+                borg_exit_codes=borg_exit_codes,
+            )
+        )
 
-    flags.warn_for_aggressive_archive_flags(json_command, json_info)
+    output_lines = tuple(
-
+        execute_command_and_capture_output(
-    execute_command(
+            main_command,
-        main_command,
+            output_log_level=logging.ANSWER,
-        output_log_level=logging.ANSWER,
+            environment=environment.make_environment(config),
-        extra_environment=environment.make_environment(config),
+            working_directory=working_directory,
-        working_directory=working_directory,
+            borg_local_path=local_path,
-        borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
-        borg_exit_codes=borg_exit_codes,
+        )
     )
+
+    flags.warn_for_aggressive_archive_flags(main_command, output_lines)
+
+    return None

borgmatic/borg/list.py

@@ -1,7 +1,9 @@
 import argparse
 import copy
+import json
 import logging
 import re
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -17,7 +19,10 @@ MAKE_FLAGS_EXCLUDES = (
     'archive',
     'paths',
     'find_paths',
-) + ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST
+    'format',
+    'json',
+    *ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST,
+)
 
 
 def make_list_command(
@@ -34,6 +39,8 @@ def make_list_command(
     and local and remote Borg paths, return a command as a tuple to list archives or paths within an
     archive.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('list', '')
+
     return (
         (local_path, 'list')
         + (
@@ -48,12 +55,17 @@ def make_list_command(
         )
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
+        + ('--log-json',)
+        + flags.make_flags('json-lines', list_arguments.json)
         + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + flags.make_flags('format', list_arguments.format or config.get('file_list_format'))
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (
             flags.make_repository_archive_flags(
-                repository_path, list_arguments.archive, local_borg_version
+                repository_path,
+                list_arguments.archive,
+                local_borg_version,
             )
             if list_arguments.archive
             else flags.make_repository_flags(repository_path, local_borg_version)
@@ -100,16 +112,15 @@ def capture_archive_listing(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, an archive name, a configuration
+    Given a local or remote repository path, an archive name, a configuration dict, the local Borg
-    dict, the local Borg version, global arguments as an argparse.Namespace,
+    version, global arguments as an argparse.Namespace, the archive paths (or Borg patterns) in
-    the archive paths (or Borg patterns) in which to list files, the Borg path
+    which to list files, the Borg path format indicating keys to include in the output, and local
-    format to use for the output, and local and remote Borg paths, capture the
+    and remote Borg paths, capture the output of listing that archive and return it as a generator
-    output of listing that archive and return it as a list of file paths.
+    of dicts, one per path.
     '''
-    borg_environment = environment.make_environment(config)
+    return (
-
+        json.loads(entry)
-    return tuple(
+        for entry in execute_command_and_capture_output(
-        execute_command_and_capture_output(
             make_list_command(
                 repository_path,
                 config,
@@ -117,22 +128,20 @@ def capture_archive_listing(
                 argparse.Namespace(
                     repository=repository_path,
                     archive=archive,
-                    paths=[path for path in list_paths] if list_paths else None,
+                    paths=list(list_paths) if list_paths else None,
                     find_paths=None,
-                    json=None,
+                    json=True,
-                    format=path_format or '{path}{NUL}',  # noqa: FS003
+                    format=path_format or None,
                 ),
                 global_arguments,
                 local_path,
                 remote_path,
             ),
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
         )
-        .strip('\n')
-        .split('\0')
     )
 
 
@@ -158,7 +167,7 @@ def list_archive(
     if not list_arguments.archive and not list_arguments.find_paths:
         if feature.available(feature.Feature.REPO_LIST, local_borg_version):
             logger.warning(
-                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.'
+                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.',
             )
 
         repo_list_arguments = argparse.Namespace(
@@ -186,15 +195,9 @@ def list_archive(
         for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
             if getattr(list_arguments, name, None):
                 logger.warning(
-                    f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag."
+                    f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag.",
                 )
 
-    if list_arguments.json:
-        raise ValueError(
-            'The --json flag on the list action is not supported when using the --archive/--find flags.'
-        )
-
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     # If there are any paths to find (and there's not a single archive already selected), start by
@@ -213,31 +216,27 @@ def list_archive(
         )
 
         # Ask Borg to list archives. Capture its output for use below.
-        archive_lines = tuple(
+        archive_lines = execute_command_and_capture_output(
-            execute_command_and_capture_output(
+            repo_list.make_repo_list_command(
-                repo_list.make_repo_list_command(
+                repository_path,
-                    repository_path,
+                config,
-                    config,
+                local_borg_version,
-                    local_borg_version,
+                repo_list_arguments,
-                    repo_list_arguments,
+                global_arguments,
-                    global_arguments,
+                local_path,
-                    local_path,
+                remote_path,
-                    remote_path,
+            ),
-                ),
+            environment=environment.make_environment(config),
-                extra_environment=borg_environment,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
-                working_directory=borgmatic.config.paths.get_working_directory(config),
+            borg_local_path=local_path,
-                borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
-                borg_exit_codes=borg_exit_codes,
-            )
-            .strip('\n')
-            .splitlines()
         )
     else:
         archive_lines = (list_arguments.archive,)
 
     # For each archive listed by Borg, run list on the contents of that archive.
     for archive in archive_lines:
-        logger.answer(f'{repository_path}: Listing archive {archive}')
+        logger.answer(f'Listing archive {archive}')
 
         archive_arguments = copy.copy(list_arguments)
         archive_arguments.archive = archive
@@ -260,8 +259,10 @@ def list_archive(
         execute_command(
             main_command,
             output_log_level=logging.ANSWER,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
+
+    return None

borgmatic/borg/mount.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags
@@ -23,14 +24,15 @@ def mount_archive(
     dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional
     local and remote Borg paths, mount the archive onto the mount point.
     '''
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
+    extra_borg_options = config.get('extra_borg_options', {}).get('mount', '')
 
     full_command = (
         (local_path, 'mount')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--log-json',) if not mount_arguments.foreground else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
@@ -39,6 +41,7 @@ def mount_archive(
             excludes=('repository', 'archive', 'mount_point', 'paths', 'options'),
         )
         + (('-o', mount_arguments.options) if mount_arguments.options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (
             (
                 flags.make_repository_flags(repository_path, local_borg_version)
@@ -59,7 +62,6 @@ def mount_archive(
         + (tuple(mount_arguments.paths) if mount_arguments.paths else ())
     )
 
-    borg_environment = environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
@@ -67,7 +69,7 @@ def mount_archive(
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
@@ -76,7 +78,7 @@ def mount_archive(
 
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/passcommand.py

@@ -0,0 +1,42 @@
+import functools
+import logging
+import shlex
+
+import borgmatic.config.paths
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+@functools.cache
+def run_passcommand(passcommand, working_directory):
+    '''
+    Run the given passcommand using the given working directory and return the passphrase produced
+    by the command.
+
+    Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
+    per borgmatic invocation.
+    '''
+    return '\n'.join(
+        borgmatic.execute.execute_command_and_capture_output(
+            shlex.split(passcommand),
+            working_directory=working_directory,
+        )
+    )
+
+
+def get_passphrase_from_passcommand(config):
+    '''
+    Given the configuration dict, call the configured passcommand to produce and return an
+    encryption passphrase. In effect, we're doing an end-run around Borg by invoking its passcommand
+    ourselves. This allows us to pass the resulting passphrase to multiple different Borg
+    invocations without the user having to be prompted multiple times.
+
+    If no passcommand is configured, then return None.
+    '''
+    passcommand = config.get('encryption_passcommand')
+
+    if not passcommand:
+        return None
+
+    return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config))

borgmatic/borg/pattern.py

@@ -1,5 +1,10 @@
 import collections
 import enum
+import logging
+import os
+import tempfile
+
+logger = logging.getLogger(__name__)
 
 
 # See https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns
@@ -20,12 +25,82 @@ class Pattern_style(enum.Enum):
     PATH_FULL_MATCH = 'pf'
 
 
+class Pattern_source(enum.Enum):
+    '''
+    Where the pattern came from within borgmatic. This is important because certain use cases (like
+    filesystem snapshotting) only want to consider patterns that the user actually put in a
+    configuration file and not patterns from other sources.
+    '''
+
+    # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories".
+    CONFIG = 'config'
+
+    # The pattern is generated internally within borgmatic, e.g. for special file excludes.
+    INTERNAL = 'internal'
+
+    # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump
+    # directory.
+    HOOK = 'hook'
+
+
 Pattern = collections.namedtuple(
     'Pattern',
-    ('path', 'type', 'style', 'device'),
+    ('path', 'type', 'style', 'device', 'source'),
     defaults=(
         Pattern_type.ROOT,
         Pattern_style.NONE,
         None,
+        Pattern_source.HOOK,
     ),
 )
+
+
+def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
+    '''
+    Given a sequence of patterns as Pattern instances, write them to a named temporary file in the
+    given borgmatic runtime directory and return the file object so it can continue to exist on disk
+    as long as the caller needs it.
+
+    If an optional open pattern file is given, append to it instead of making a new temporary file.
+    Return None if no patterns are provided.
+    '''
+    if not patterns:
+        return None
+
+    if patterns_file is None:
+        patterns_file = tempfile.NamedTemporaryFile(
+            'w', dir=borgmatic_runtime_directory, encoding='utf-8'
+        )
+        operation_name = 'Writing'
+    else:
+        patterns_file.write('\n')
+        operation_name = 'Appending'
+
+    patterns_output = '\n'.join(
+        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
+        for pattern in patterns
+    )
+    logger.debug(f'{operation_name} patterns to {patterns_file.name}:\n{patterns_output}')
+
+    patterns_file.write(patterns_output)
+    patterns_file.flush()
+
+    return patterns_file
+
+
+def check_all_root_patterns_exist(patterns, working_directory):
+    '''
+    Given a sequence of Pattern instances and the current working directory, check that all root
+    pattern paths exist. If any don't, raise an exception.
+    '''
+    missing_paths = [
+        pattern.path
+        for pattern in patterns
+        if pattern.type == Pattern_type.ROOT
+        if not os.path.exists(os.path.join(working_directory or '', pattern.path))
+    ]
+
+    if missing_paths:
+        raise ValueError(
+            f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}",
+        )

borgmatic/borg/prune.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -41,7 +42,7 @@ def make_prune_flags(config, prune_arguments, local_borg_version):
         if prefix
         else (
             flags.make_match_archives_flags(
-                prune_arguments.match_archives or config.get('match_archives'),
+                config.get('match_archives'),
                 config.get('archive_name_format'),
                 local_borg_version,
             )
@@ -64,31 +65,50 @@ def prune_archives(
     archives according to the retention policy specified in that configuration.
     '''
     borgmatic.logger.add_custom_log_levels()
-    umask = config.get('umask', None)
+    umask = config.get('umask')
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
     extra_borg_options = config.get('extra_borg_options', {}).get('prune', '')
 
     full_command = (
         (local_path, 'prune')
         + make_prune_flags(config, prune_arguments, local_borg_version)
+        + ('--log-json',)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--stats',) if prune_arguments.stats and not dry_run else ())
+        + (
+            ('--stats',)
+            if config.get('statistics')
+            and not dry_run
+            and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version)
+            else ()
+        )
+        + (
+            ('--quick-stats',)
+            if config.get('quick_statistics')
+            and not dry_run
+            and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version)
+            else ()
+        )
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + flags.make_flags_from_arguments(
             prune_arguments,
-            excludes=('repository', 'match_archives', 'stats', 'list_archives'),
+            excludes=(
+                'repository',
+                'match_archives',
+                'statistics',
+                'quick_statistics',
+                'list_details',
+            ),
         )
-        + (('--list',) if prune_arguments.list_archives else ())
+        + (('--list',) if config.get('list_details') else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    if prune_arguments.stats or prune_arguments.list_archives:
+    if config.get('statistics') or config.get('quick_statistics') or config.get('list_details'):
         output_log_level = logging.ANSWER
     else:
         output_log_level = logging.INFO
@@ -96,7 +116,7 @@ def prune_archives(
     execute_command(
         full_command,
         output_log_level=output_log_level,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/recreate.py

@@ -0,0 +1,105 @@
+import logging
+import shlex
+
+import borgmatic.borg.environment
+import borgmatic.borg.feature
+import borgmatic.config.paths
+import borgmatic.execute
+from borgmatic.borg import flags
+from borgmatic.borg.pattern import write_patterns_file
+
+logger = logging.getLogger(__name__)
+
+
+def recreate_archive(
+    repository,
+    archive,
+    config,
+    local_borg_version,
+    recreate_arguments,
+    global_arguments,
+    local_path,
+    remote_path=None,
+    patterns=None,
+):
+    '''
+    Given a local or remote repository path, an archive name, a configuration dict, the local Borg
+    version string, an argparse.Namespace of recreate arguments, an argparse.Namespace of global
+    arguments, optional local and remote Borg paths, executes the recreate command with the given
+    arguments.
+    '''
+    lock_wait = config.get('lock_wait')
+    exclude_flags = flags.make_exclude_flags(config)
+    compression = config.get('compression')
+    chunker_params = config.get('chunker_params')
+    extra_borg_options = config.get('extra_borg_options', {}).get('recreate', '')
+
+    # Available recompress MODES: "if-different", "always", "never" (default)
+    recompress = config.get('recompress')
+
+    # Write patterns to a temporary file and use that file with --patterns-from.
+    patterns_file = write_patterns_file(
+        patterns,
+        borgmatic.config.paths.get_working_directory(config),
+    )
+
+    recreate_command = (
+        (local_path, 'recreate')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + ('--log-json',)
+        + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--patterns-from', patterns_file.name) if patterns_file else ())
+        + (
+            (
+                '--list',
+                '--filter',
+                flags.make_list_filter_flags(local_borg_version, global_arguments.dry_run),
+            )
+            if config.get('list_details')
+            else ()
+        )
+        # Flag --target works only for a single archive.
+        + (('--target', recreate_arguments.target) if recreate_arguments.target and archive else ())
+        + (
+            ('--comment', shlex.quote(recreate_arguments.comment))
+            if recreate_arguments.comment
+            else ()
+        )
+        + (('--timestamp', recreate_arguments.timestamp) if recreate_arguments.timestamp else ())
+        + (('--compression', compression) if compression else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
+        + (('--recompress', recompress) if recompress else ())
+        + exclude_flags
+        + (('--dry-run',) if global_arguments.dry_run else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + (
+            (
+                flags.make_repository_flags(repository, local_borg_version)
+                + flags.make_match_archives_flags(
+                    archive or config.get('match_archives'),
+                    config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE,
+                local_borg_version,
+            )
+            else (
+                flags.make_repository_archive_flags(repository, archive, local_borg_version)
+                if archive
+                else flags.make_repository_flags(repository, local_borg_version)
+            )
+        )
+    )
+
+    borgmatic.execute.execute_command(
+        full_command=recreate_command,
+        output_log_level=logging.INFO,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/rename.py

@@ -0,0 +1,69 @@
+import logging
+import shlex
+
+import borgmatic.borg.environment
+import borgmatic.borg.flags
+
+logger = logging.getLogger(__name__)
+
+
+def make_rename_command(
+    dry_run,
+    repository_name,
+    old_archive_name,
+    new_archive_name,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    extra_borg_options = config.get('extra_borg_options', {}).get('rename', '')
+
+    return (
+        (local_path, 'rename')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + borgmatic.borg.flags.make_flags('dry-run', dry_run)
+        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
+        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
+        + ('--log-json',)
+        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + borgmatic.borg.flags.make_repository_archive_flags(
+            repository_name,
+            old_archive_name,
+            local_borg_version,
+        )
+        + (new_archive_name,)
+    )
+
+
+def rename_archive(
+    repository_name,
+    old_archive_name,
+    new_archive_name,
+    dry_run,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    command = make_rename_command(
+        dry_run,
+        repository_name,
+        old_archive_name,
+        new_archive_name,
+        config,
+        local_borg_version,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.execute.execute_command(
+        command,
+        output_log_level=logging.INFO,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/repo_create.py

@@ -1,9 +1,11 @@
 import argparse
 import json
 import logging
+import shlex
 import subprocess
 
 import borgmatic.config.paths
+import borgmatic.logger
 from borgmatic.borg import environment, feature, flags, repo_info
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
@@ -24,7 +26,7 @@ def create_repository(
     copy_crypt_key=False,
     append_only=None,
     storage_quota=None,
-    make_parent_dirs=False,
+    make_parent_directories=False,
     local_path='borg',
     remote_path=None,
 ):
@@ -39,25 +41,29 @@ def create_repository(
     Raise subprocess.CalledProcessError if "borg info" returns an error exit code.
     '''
     try:
-        info_data = json.loads(
+        # Suppress Borg's "repository does not exist" error log, so the user isn't confused by
-            repo_info.display_repository_info(
+        # seeing an error during successful repository creation.
-                repository_path,
+        with borgmatic.logger.Logs_suppressed(msgid='Repository.DoesNotExist'):
-                config,
+            info_data = json.loads(
-                local_borg_version,
+                repo_info.display_repository_info(
-                argparse.Namespace(json=True),
+                    repository_path,
-                global_arguments,
+                    config,
-                local_path,
+                    local_borg_version,
-                remote_path,
+                    argparse.Namespace(json=True),
+                    global_arguments,
+                    local_path,
+                    remote_path,
+                ),
             )
-        )
+
         repository_encryption_mode = info_data.get('encryption', {}).get('mode')
 
         if repository_encryption_mode != encryption_mode:
             raise ValueError(
-                f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"'
+                f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"',
             )
 
-        logger.info(f'{repository_path}: Repository already exists. Skipping creation.')
+        logger.info('Repository already exists. Skipping creation.')
         return
     except subprocess.CalledProcessError as error:
         if error.returncode not in REPO_INFO_REPOSITORY_NOT_FOUND_EXIT_CODES:
@@ -65,8 +71,14 @@ def create_repository(
 
     lock_wait = config.get('lock_wait')
     umask = config.get('umask')
+    extra_borg_options_from_init = config.get('extra_borg_options', {}).get('init', '')
     extra_borg_options = config.get('extra_borg_options', {}).get('repo-create', '')
 
+    if extra_borg_options_from_init:
+        logger.warning(
+            'The "init" option in "extra_borg_options" is deprecated and will be removed from a future release; use "repo_create" instead.'
+        )
+
     repo_create_command = (
         (local_path,)
         + (
@@ -79,26 +91,26 @@ def create_repository(
         + (('--copy-crypt-key',) if copy_crypt_key else ())
         + (('--append-only',) if append_only else ())
         + (('--storage-quota', storage_quota) if storage_quota else ())
-        + (('--make-parent-dirs',) if make_parent_dirs else ())
+        + (('--make-parent-dirs',) if make_parent_directories else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options_from_init)) if extra_borg_options_from_init else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if dry_run:
-        logging.info(f'{repository_path}: Skipping repository creation (dry run)')
+        logger.info('Skipping repository creation (dry run)')
         return
 
     # Do not capture output here, so as to support interactive prompts.
     execute_command(
         repo_create_command,
         output_file=DO_NOT_CAPTURE,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_delete.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.borg.environment
 import borgmatic.borg.feature
@@ -9,6 +10,9 @@ import borgmatic.execute
 logger = logging.getLogger(__name__)
 
 
+FORCE_HARDER_FLAG_COUNT = 2
+
+
 def make_repo_delete_command(
     repository,
     config,
@@ -17,18 +21,30 @@ def make_repo_delete_command(
     global_arguments,
     local_path,
     remote_path,
+    output_file,
 ):
     '''
     Given a local or remote repository dict, a configuration dict, the local Borg version, the
-    arguments to the repo_delete action as an argparse.Namespace, and global arguments, return a command
+    arguments to the repo_delete action as an argparse.Namespace, and global arguments, the Borg
-    as a tuple to repo_delete the entire repository.
+    local path, the Borg remote path, and an optional output file, return a command as a tuple to
+    repo_delete the entire repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_delete'
+        if borgmatic.borg.feature.available(
+            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+        )
+        else 'delete',
+        '',
+    )
+
     return (
         (local_path,)
         + (
             ('repo-delete',)
             if borgmatic.borg.feature.available(
-                borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+                borgmatic.borg.feature.Feature.REPO_DELETE,
+                local_borg_version,
             )
             else ('delete',)
         )
@@ -37,17 +53,22 @@ def make_repo_delete_command(
         + borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
         + borgmatic.borg.flags.make_flags('remote-path', remote_path)
         + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
-        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+        + (('--log-json',) if output_file is None else ())
         + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
-        + borgmatic.borg.flags.make_flags('list', repo_delete_arguments.list_archives)
+        + borgmatic.borg.flags.make_flags('list', config.get('list_details'))
         + (
-            (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ()))
+            (
+                ('--force',)
+                + (('--force',) if repo_delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
+            )
             if repo_delete_arguments.force
             else ()
         )
         + borgmatic.borg.flags.make_flags_from_arguments(
-            repo_delete_arguments, excludes=('list_archives', 'force', 'repository')
+            repo_delete_arguments,
+            excludes=('list_details', 'force', 'repository'),
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
     )
 
@@ -68,6 +89,14 @@ def delete_repository(
     '''
     borgmatic.logger.add_custom_log_levels()
 
+    # Don't capture output when Borg is expected to prompt for interactive confirmation, or the
+    # prompt won't work.
+    output_file = (
+        None
+        if repo_delete_arguments.force or repo_delete_arguments.cache_only
+        else borgmatic.execute.DO_NOT_CAPTURE
+    )
+
     command = make_repo_delete_command(
         repository,
         config,
@@ -76,19 +105,14 @@ def delete_repository(
         global_arguments,
         local_path,
         remote_path,
+        output_file,
     )
 
     borgmatic.execute.execute_command(
         command,
         output_log_level=logging.ANSWER,
-        # Don't capture output when Borg is expected to prompt for interactive confirmation, or the
+        output_file=output_file,
-        # prompt won't work.
+        environment=borgmatic.borg.environment.make_environment(config),
-        output_file=(
-            None
-            if repo_delete_arguments.force or repo_delete_arguments.cache_only
-            else borgmatic.execute.DO_NOT_CAPTURE
-        ),
-        extra_environment=borgmatic.borg.environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_info.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -23,7 +24,11 @@ def display_repository_info(
     information for the Borg repository or return JSON summary information.
     '''
     borgmatic.logger.add_custom_log_levels()
-    lock_wait = config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait')
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_info' if feature.available(feature.Feature.REPO_INFO, local_borg_version) else 'info',
+        '',
+    )
 
     full_command = (
         (local_path,)
@@ -44,30 +49,34 @@ def display_repository_info(
         )
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
+        + ('--log-json',)
         + (('--json',) if repo_info_arguments.json else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    extra_environment = environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     if repo_info_arguments.json:
-        return execute_command_and_capture_output(
+        return '\n'.join(
-            full_command,
+            execute_command_and_capture_output(
-            extra_environment=extra_environment,
+                full_command,
-            working_directory=working_directory,
+                environment=environment.make_environment(config),
-            borg_local_path=local_path,
+                working_directory=working_directory,
-            borg_exit_codes=borg_exit_codes,
+                borg_local_path=local_path,
-        )
+                borg_exit_codes=borg_exit_codes,
-    else:
+            )
-        execute_command(
-            full_command,
-            output_log_level=logging.ANSWER,
-            extra_environment=extra_environment,
-            working_directory=working_directory,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
         )
+
+    execute_command(
+        full_command,
+        output_log_level=logging.ANSWER,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )
+
+    return None

borgmatic/borg/repo_list.py

@@ -1,10 +1,12 @@
 import argparse
+import json
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command, execute_command_and_capture_output
+from borgmatic.execute import execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
@@ -22,49 +24,102 @@ def resolve_archive_name(
     Given a local or remote repository path, an archive name, a configuration dict, the local Borg
     version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
     return the archive name. But if the archive name is "latest", then instead introspect the
-    repository for the latest archive and return its name.
+    repository for the latest archive and return its name or ID, depending on whether the version of
+    Borg in use supports archive series—different archives that share the same name but have unique
+    IDs.
 
     Raise ValueError if "latest" is given but there are no archives in the repository.
     '''
     if archive != 'latest':
         return archive
 
-    full_command = (
+    latest_archive = get_latest_archive(
-        (
+        repository_path,
-            local_path,
+        config,
-            (
+        local_borg_version,
-                'repo-list'
+        global_arguments,
-                if feature.available(feature.Feature.REPO_LIST, local_borg_version)
+        local_path=local_path,
-                else 'list'
+        remote_path=remote_path,
-            ),
-        )
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', config.get('lock_wait'))
-        + flags.make_flags('last', 1)
-        + ('--short',)
-        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    output = execute_command_and_capture_output(
+    return (
-        full_command,
+        latest_archive['id']
-        extra_environment=environment.make_environment(config),
+        if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+        else latest_archive['name']
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
     )
+
+
+def get_latest_archive(
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+    consider_checkpoints=False,
+):
+    '''
+    Returns a dict with information about the latest archive of a repository.
+
+    Raises ValueError if there are no archives in the repository.
+    '''
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_list' if feature.available(feature.Feature.REPO_LIST, local_borg_version) else 'list',
+        '',
+    )
+
+    full_command = (
+        local_path,
+        (
+            'repo-list'
+            if feature.available(feature.Feature.REPO_LIST, local_borg_version)
+            else 'list'
+        ),
+        *flags.make_flags('remote-path', remote_path),
+        *flags.make_flags('umask', config.get('umask')),
+        *('--log-json',),
+        *flags.make_flags('lock-wait', config.get('lock_wait')),
+        *(
+            flags.make_match_archives_flags(
+                config.get('match_archives'),
+                config.get('archive_name_format'),
+                local_borg_version,
+            )
+        ),
+        *(
+            flags.make_flags('consider-checkpoints', consider_checkpoints)
+            if not feature.available(feature.Feature.REPO_LIST, local_borg_version)
+            else ()
+        ),
+        *flags.make_flags('last', 1),
+        '--json',
+        *(tuple(shlex.split(extra_borg_options)) if extra_borg_options else ()),
+        *flags.make_repository_flags(repository_path, local_borg_version),
+    )
+
+    json_output = '\n'.join(
+        execute_command_and_capture_output(
+            full_command,
+            environment=environment.make_environment(config),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
+        )
+    )
+
+    archives = json.loads(json_output)['archives']
+
     try:
-        latest_archive = output.strip().splitlines()[-1]
+        latest_archive = archives[-1]
     except IndexError:
         raise ValueError('No archives found in the repository')
 
-    logger.debug(f'{repository_path}: Latest archive is {latest_archive}')
+    logger.debug(f'Latest archive is {latest_archive["name"]} ({latest_archive["id"]})')
 
     return latest_archive
 
 
-MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
+MAKE_FLAGS_EXCLUDES = ('repository', 'format', 'prefix', 'match_archives')
 
 
 def make_repo_list_command(
@@ -81,6 +136,11 @@ def make_repo_list_command(
     arguments to the repo_list action, global arguments as an argparse.Namespace instance, and local and
     remote Borg paths, return a command as a tuple to list archives with a repository.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get(
+        'repo_list' if feature.available(feature.Feature.REPO_LIST, local_borg_version) else 'list',
+        '',
+    )
+
     return (
         (
             local_path,
@@ -102,7 +162,7 @@ def make_repo_list_command(
         )
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
+        + ('--log-json',)
         + flags.make_flags('lock-wait', config.get('lock_wait'))
         + (
             (
@@ -113,13 +173,17 @@ def make_repo_list_command(
             if repo_list_arguments.prefix
             else (
                 flags.make_match_archives_flags(
-                    repo_list_arguments.match_archives or config.get('match_archives'),
+                    config.get('match_archives'),
                     config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
         )
+        + flags.make_flags(
+            'format', repo_list_arguments.format or config.get('archive_list_format')
+        )
         + flags.make_flags_from_arguments(repo_list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
@@ -140,7 +204,6 @@ def list_repository(
     return JSON output).
     '''
     borgmatic.logger.add_custom_log_levels()
-    borg_environment = environment.make_environment(config)
 
     main_command = make_repo_list_command(
         repository_path,
@@ -163,24 +226,28 @@ def list_repository(
     working_directory = borgmatic.config.paths.get_working_directory(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
-    json_listing = execute_command_and_capture_output(
-        json_command,
-        extra_environment=borg_environment,
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=borg_exit_codes,
-    )
-
     if repo_list_arguments.json:
-        return json_listing
+        return '\n'.join(
+            execute_command_and_capture_output(
+                json_command,
+                environment=environment.make_environment(config),
+                working_directory=working_directory,
+                borg_local_path=local_path,
+                borg_exit_codes=borg_exit_codes,
+            )
+        )
 
-    flags.warn_for_aggressive_archive_flags(json_command, json_listing)
+    output_lines = tuple(
-
+        execute_command_and_capture_output(
-    execute_command(
+            main_command,
-        main_command,
+            output_log_level=logging.ANSWER,
-        output_log_level=logging.ANSWER,
+            environment=environment.make_environment(config),
-        extra_environment=borg_environment,
+            working_directory=working_directory,
-        working_directory=working_directory,
+            borg_local_path=local_path,
-        borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
-        borg_exit_codes=borg_exit_codes,
+        )
     )
+
+    flags.warn_for_aggressive_archive_flags(main_command, output_lines)
+
+    return None

borgmatic/borg/state.py

@@ -1 +1 @@
-DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
+DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'  # pragma: nocover

borgmatic/borg/transfer.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -24,6 +25,7 @@ def transfer_archives(
     instance, transfer archives to the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
+    extra_borg_options = config.get('extra_borg_options', {}).get('transfer', '')
 
     full_command = (
         (local_path, 'transfer')
@@ -31,23 +33,29 @@ def transfer_archives(
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', global_arguments.log_json)
+        + (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
-        + flags.make_flags('lock-wait', config.get('lock_wait', None))
+        + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + flags.make_flags('progress', config.get('progress'))
         + (
             flags.make_flags_from_arguments(
                 transfer_arguments,
-                excludes=('repository', 'source_repository', 'archive', 'match_archives'),
+                excludes=(
+                    'repository',
+                    'source_repository',
+                    'archive',
+                    'match_archives',
+                    'progress',
+                ),
             )
             or (
                 flags.make_match_archives_flags(
-                    transfer_arguments.match_archives
+                    transfer_arguments.archive or config.get('match_archives'),
-                    or transfer_arguments.archive
-                    or config.get('match_archives'),
                     config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
         )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
         + flags.make_flags('other-repo', transfer_arguments.source_repository)
         + flags.make_flags('dry-run', dry_run)
@@ -56,8 +64,8 @@ def transfer_archives(
     return execute_command(
         full_command,
         output_log_level=logging.ANSWER,
-        output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
+        output_file=DO_NOT_CAPTURE if config.get('progress') else None,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/umount.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.execute import execute_command
@@ -11,10 +12,13 @@ def unmount_archive(config, mount_point, local_path='borg'):
     Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
     from the mount point.
     '''
+    extra_borg_options = config.get('extra_borg_options', {}).get('umount', '')
     full_command = (
         (local_path, 'umount')
+        + ('--log-json',)
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (mount_point,)
     )
 

borgmatic/borg/version.py

@@ -16,15 +16,19 @@ def local_borg_version(config, local_path='borg'):
     '''
     full_command = (
         (local_path, '--version')
+        + ('--log-json',)
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
     )
-    output = execute_command_and_capture_output(
+
-        full_command,
+    output = '\n'.join(
-        extra_environment=environment.make_environment(config),
+        execute_command_and_capture_output(
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+            full_command,
-        borg_local_path=local_path,
+            environment=environment.make_environment(config),
-        borg_exit_codes=config.get('borg_exit_codes'),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
+        )
     )
 
     try:

borgmatic/commands/completion/actions.py

@@ -22,15 +22,15 @@ def available_actions(subparsers, current_action=None):
     action of "config" but not "list".
     '''
     action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions(
-        subparsers.choices
+        subparsers.choices,
     )
     current_subactions = action_to_subactions.get(current_action)
 
     if current_subactions:
         return current_subactions
 
-    all_subactions = set(
+    all_subactions = {
         subaction for subactions in action_to_subactions.values() for subaction in subactions
-    )
+    }
 
-    return tuple(action for action in subparsers.choices.keys() if action not in all_subactions)
+    return tuple(action for action in subparsers.choices if action not in all_subactions)

borgmatic/commands/completion/bash.py

@@ -1,5 +1,7 @@
 import borgmatic.commands.arguments
 import borgmatic.commands.completion.actions
+import borgmatic.commands.completion.flag
+import borgmatic.config.validate
 
 
 def parser_flags(parser):
@@ -7,7 +9,12 @@ def parser_flags(parser):
     Given an argparse.ArgumentParser instance, return its argument flags in a space-separated
     string.
     '''
-    return ' '.join(option for action in parser._actions for option in action.option_strings)
+    return ' '.join(
+        flag_variant
+        for action in parser._actions
+        for flag_name in action.option_strings
+        for flag_variant in borgmatic.commands.completion.flag.variants(flag_name)
+    )
 
 
 def bash_completion():
@@ -16,19 +23,23 @@ def bash_completion():
     borgmatic's command-line argument parsers.
     '''
     (
-        unused_global_parser,
+        _,
         action_parsers,
         global_plus_action_parser,
-    ) = borgmatic.commands.arguments.make_parsers()
+    ) = borgmatic.commands.arguments.make_parsers(
+        schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()),
+        unparsed_arguments=(),
+    )
     global_flags = parser_flags(global_plus_action_parser)
 
     # Avert your eyes.
+    # fmt: off
     return '\n'.join(
         (
             'check_version() {',
             '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
             '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
-            '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
+            '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];',
             f'''        then cat << EOF\n{borgmatic.commands.completion.actions.upgrade_message(
                     'bash',
                     'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
@@ -37,24 +48,22 @@ def bash_completion():
             '    fi',
             '}',
             'complete_borgmatic() {',
-        )
+            *tuple(
-        + tuple(
             '''    if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
         COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
         return 0
-    fi'''
+    fi'''  # noqa: UP031
             % (
                 action,
                 parser_flags(action_parser),
                 ' '.join(
-                    borgmatic.commands.completion.actions.available_actions(action_parsers, action)
+                    borgmatic.commands.completion.actions.available_actions(action_parsers, action),
                 ),
                 global_flags,
             )
             for action, action_parser in reversed(action_parsers.choices.items())
-        )
+        ),
-        + (
+            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: UP031
-            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
             % (
                 ' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)),
                 global_flags,
@@ -62,5 +71,5 @@ def bash_completion():
             '    (check_version &)',
             '}',
             '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
-        )
+        ),
     )

borgmatic/commands/completion/fish.py

@@ -4,16 +4,17 @@ from textwrap import dedent
 
 import borgmatic.commands.arguments
 import borgmatic.commands.completion.actions
+import borgmatic.config.validate
 
 
 def has_file_options(action: Action):
     '''
     Given an argparse.Action instance, return True if it takes a file argument.
     '''
-    return action.metavar in (
+    return action.metavar in {
         'FILENAME',
         'PATH',
-    ) or action.dest in ('config_paths',)
+    } or action.dest in {'config_paths'}
 
 
 def has_choice_options(action: Action):
@@ -26,18 +27,20 @@ def has_choice_options(action: Action):
 def has_unknown_required_param_options(action: Action):
     '''
     A catch-all for options that take a required parameter, but we don't know what the parameter is.
-    This should be used last. These are actions that take something like a glob, a list of numbers, or a string.
+    This should be used last. These are actions that take something like a glob, a list of numbers,
+    or a string.
 
-    Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid.
+    Actions that match this pattern should not show the normal arguments, because those are unlikely
+    to be valid.
     '''
     return (
         action.required is True
         or action.nargs
-        in (
+        in {
             '+',
             '*',
-        )
+        }
-        or action.metavar in ('PATTERN', 'KEYS', 'N')
+        or action.metavar in {'PATTERN', 'KEYS', 'N'}
         or (action.type is not None and action.default is None)
     )
 
@@ -52,9 +55,9 @@ def has_exact_options(action: Action):
 
 def exact_options_completion(action: Action):
     '''
-    Given an argparse.Action instance, return a completion invocation that forces file completions, options completion,
+    Given an argparse.Action instance, return a completion invocation that forces file completions,
-    or just that some value follow the action, if the action takes such an argument and was the last action on the
+    options completion, or just that some value follow the action, if the action takes such an
-    command line prior to the cursor.
+    argument and was the last action on the command line prior to the cursor.
 
     Otherwise, return an empty string.
     '''
@@ -74,14 +77,15 @@ def exact_options_completion(action: Action):
         return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"'''
 
     raise ValueError(
-        f'Unexpected action: {action} passes has_exact_options but has no choices produced'
+        f'Unexpected action: {action} passes has_exact_options but has no choices produced',
     )
 
 
 def dedent_strip_as_tuple(string: str):
     '''
-    Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string.
+    Dedent a string, then strip it to avoid requiring your first line to have content, then return a
-    Makes it easier to write multiline strings for completions when you join them with a tuple.
+    tuple of the string. Makes it easier to write multiline strings for completions when you join
+    them with a tuple.
     '''
     return (dedent(string).strip('\n'),)
 
@@ -92,12 +96,15 @@ def fish_completion():
     borgmatic's command-line argument parsers.
     '''
     (
-        unused_global_parser,
+        _,
         action_parsers,
         global_plus_action_parser,
-    ) = borgmatic.commands.arguments.make_parsers()
+    ) = borgmatic.commands.arguments.make_parsers(
+        schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()),
+        unparsed_arguments=(),
+    )
 
-    all_action_parsers = ' '.join(action for action in action_parsers.choices.keys())
+    all_action_parsers = ' '.join(action for action in action_parsers.choices)
 
     exact_option_args = tuple(
         ' '.join(action.option_strings)
@@ -112,8 +119,9 @@ def fish_completion():
     )
 
     # Avert your eyes.
-    return '\n'.join(
+    # fmt: off
-        dedent_strip_as_tuple(
+    return '\n'.join((
+        *dedent_strip_as_tuple(
             f'''
             function __borgmatic_check_version
                 set -fx this_filename (status current-filename)
@@ -150,27 +158,27 @@ def fish_completion():
 
             set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}"
             set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
-            '''
+            ''',
-        )
+        ),
-        + ('\n# action_parser completions',)
+        '\n# action_parser completions',
-        + tuple(
+        *tuple(
             f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}'''
             for action_name, action_parser in action_parsers.choices.items()
-        )
+        ),
-        + ('\n# global flags',)
+        '\n# global flags',
-        + tuple(
+        *tuple(
             # -n is checked in order, so put faster / more likely to be true checks first
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
             for action in global_plus_action_parser._actions
             # ignore the noargs action, as this is an impossible completion for fish
             if len(action.option_strings) > 0
             if 'Deprecated' not in action.help
-        )
+        ),
-        + ('\n# action_parser flags',)
+        '\n# action_parser flags',
-        + tuple(
+        *tuple(
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
             for action_name, action_parser in action_parsers.choices.items()
             for action in action_parser._actions
             if 'Deprecated' not in (action.help or ())
-        )
+        ),
-    )
+    ))

borgmatic/commands/completion/flag.py

@@ -0,0 +1,13 @@
+def variants(flag_name):
+    '''
+    Given a flag name as a string, yield it and any variations that should be complete-able as well.
+    For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ...,
+    "--foo[9].bar".
+    '''
+    if '[0]' in flag_name:
+        for index in range(10):
+            yield flag_name.replace('[0]', f'[{index}]')
+
+        return
+
+    yield flag_name

borgmatic/commands/generate_config.py

@@ -10,8 +10,8 @@ def main():
             levelno=logging.WARNING,
             levelname='WARNING',
             msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
-        )
+        ),
     )
 
-    sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
+    sys.argv = ['borgmatic', 'config', 'generate', *sys.argv[1:]]
     borgmatic.commands.borgmatic.main([warning_log])

borgmatic/commands/validate_config.py

@@ -10,8 +10,8 @@ def main():
             levelno=logging.WARNING,
             levelname='WARNING',
             msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.',
-        )
+        ),
     )
 
-    sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:]
+    sys.argv = ['borgmatic', 'config', 'validate', *sys.argv[1:]]
     borgmatic.commands.borgmatic.main([warning_log])

borgmatic/config/arguments.py

@@ -0,0 +1,176 @@
+import io
+import re
+
+import ruamel.yaml
+
+import borgmatic.config.schema
+
+LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P<list_name>[a-zA-z-]+)\[(?P<index>\d+)\]$')
+
+
+def set_values(config, keys, value):
+    '''
+    Given a configuration dict, a sequence of parsed key strings, and a string value, descend into
+    the configuration hierarchy based on the given keys and set the value into the right place.
+    For example, consider these keys:
+
+        ('foo', 'bar', 'baz')
+
+    This looks up "foo" in the given configuration dict. And within that, it looks up "bar". And
+    then within that, it looks up "baz" and sets it to the given value. Another example:
+
+        ('mylist[0]', 'foo')
+
+    This looks for the zeroth element of "mylist" in the given configuration. And within that, it
+    looks up "foo" and sets it to the given value.
+    '''
+    if not keys:
+        return
+
+    first_key = keys[0]
+
+    # Support "mylist[0]" list index syntax.
+    match = LIST_INDEX_KEY_PATTERN.match(first_key)
+
+    if match:
+        list_key = match.group('list_name')
+        list_index = int(match.group('index'))
+
+        try:
+            if len(keys) == 1:
+                config[list_key][list_index] = value
+
+                return
+
+            if list_key not in config:
+                config[list_key] = []
+
+            set_values(config[list_key][list_index], keys[1:], value)
+        except (IndexError, KeyError):
+            raise ValueError(f'Argument list index {first_key} is out of range')
+
+        return
+
+    if len(keys) == 1:
+        config[first_key] = value
+
+        return
+
+    if first_key not in config:
+        config[first_key] = {}
+
+    set_values(config[first_key], keys[1:], value)
+
+
+def type_for_option(schema, option_keys):
+    '''
+    Given a configuration schema dict and a sequence of keys identifying a potentially nested
+    option, e.g. ('extra_borg_options', 'create'), return the schema type of that option as a
+    string.
+
+    Return None if the option or its type cannot be found in the schema.
+    '''
+    option_schema = schema
+
+    for key in option_keys:
+        # Support "name[0]"-style list index syntax.
+        match = LIST_INDEX_KEY_PATTERN.match(key)
+        properties = borgmatic.config.schema.get_properties(option_schema)
+
+        try:
+            if match:
+                option_schema = properties[match.group('list_name')]['items']
+            else:
+                option_schema = properties[key]
+        except KeyError:
+            return None
+
+    try:
+        return option_schema['type']
+    except KeyError:
+        return None
+
+
+def convert_value_type(value, option_type):
+    '''
+    Given a string value and its schema type as a string, determine its logical type (string,
+    boolean, integer, etc.), and return it converted to that type.
+
+    If the destination option type is a string, then leave the value as-is so that special
+    characters in it don't get interpreted as YAML during conversion.
+
+    And if the source value isn't a string, return it as-is.
+
+    Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML.
+    Raise ValueError if the parsed value doesn't match the option type.
+    '''
+    if not isinstance(value, str):
+        return value
+
+    if option_type == 'string':
+        return value
+
+    try:
+        parsed_value = ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
+    except ruamel.yaml.error.YAMLError as error:
+        raise ValueError(f'Argument value "{value}" is invalid: {error.problem}')
+
+    if not isinstance(parsed_value, borgmatic.config.schema.parse_type(option_type)):
+        raise ValueError(f'Argument value "{value}" is not of the expected type: {option_type}')
+
+    return parsed_value
+
+
+def prepare_arguments_for_config(global_arguments, schema):
+    '''
+    Given global arguments as an argparse.Namespace and a configuration schema dict, parse each
+    argument that corresponds to an option in the schema and return a sequence of tuples (keys,
+    values) for that option, where keys is a sequence of strings. For instance, given the following
+    arguments:
+
+        argparse.Namespace(**{'my_option.sub_option': 'value1', 'other_option': 'value2'})
+
+    ... return this:
+
+        (
+            (('my_option', 'sub_option'), 'value1'),
+            (('other_option',), 'value2'),
+        )
+    '''
+    prepared_values = []
+
+    for argument_name, value in global_arguments.__dict__.items():
+        if value is None:
+            continue
+
+        keys = tuple(argument_name.split('.'))
+        option_type = type_for_option(schema, keys)
+
+        # The argument doesn't correspond to any option in the schema, or it is a complex argument, so ignore it.
+        # It's probably a flag that borgmatic has on the command-line but not in configuration.
+        if option_type in {'object', None}:
+            continue
+
+        prepared_values.append(
+            (
+                keys,
+                convert_value_type(value, option_type),
+            ),
+        )
+
+    return tuple(prepared_values)
+
+
+def apply_arguments_to_config(config, schema, arguments):
+    '''
+    Given a configuration dict, a corresponding configuration schema dict, and arguments as a dict
+    from action name to argparse.Namespace, set those given argument values into their corresponding
+    configuration options in the configuration dict.
+
+    This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested
+    configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list
+    element in the configuration.
+    '''
+    for action_arguments in arguments.values():
+        for keys, value in prepare_arguments_for_config(action_arguments, schema):
+            set_values(config, keys, value)

borgmatic/config/collect.py

@@ -1,17 +1,14 @@
 import os
 
 
-def get_default_config_paths(expand_home=True):
+def get_default_config_paths():
     '''
-    Based on the value of the XDG_CONFIG_HOME and HOME environment variables, return a list of
+    Return a list of default configuration paths. This includes both system-wide
-    default configuration paths. This includes both system-wide configuration and configuration in
+    configuration and configuration in the current user's home directory.
-    the current user's home directory.
-
-    Don't expand the home directory ($HOME) if the expand home flag is False.
     '''
-    user_config_directory = os.getenv('XDG_CONFIG_HOME') or os.path.join('$HOME', '.config')
+    user_config_directory = os.path.expandvars(
-    if expand_home:
+        os.getenv('XDG_CONFIG_HOME') or os.path.join('$HOME', '.config')
-        user_config_directory = os.path.expandvars(user_config_directory)
+    )
 
     return [
         '/etc/borgmatic/config.yaml',
@@ -49,6 +46,7 @@ def collect_config_filenames(config_paths):
 
         for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
-            matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
+            matching_filetype = full_filename.endswith(('.yaml', '.yml'))
+
             if matching_filetype and not os.path.isdir(full_filename):
                 yield os.path.abspath(full_filename)

borgmatic/config/constants.py

@@ -1,3 +1,6 @@
+import contextlib
+import functools
+import re
 import shlex
 
 
@@ -6,20 +9,58 @@ def coerce_scalar(value):
     Given a configuration value, coerce it to an integer or a boolean as appropriate and return the
     result.
     '''
-    try:
+    with contextlib.suppress(TypeError, ValueError):
         return int(value)
-    except (TypeError, ValueError):
-        pass
 
-    if value == 'true' or value == 'True':
+    try:
-        return True
+        return {
-    if value == 'false' or value == 'False':
+            'true': True,
-        return False
+            'True': True,
-
+            'false': False,
-    return value
+            'False': False,
+        }.get(value, value)
+    except TypeError:  # e.g. for an unhashable type
+        return value
 
 
-def apply_constants(value, constants, shell_escape=False):
+CONSTANT_PATTERN = re.compile(r'(?P<left_escape>\\)?\{(?P<name>[\w]+)(?P<right_escape>\\)?\}')
+
+
+def resolve_constant(match, constants, command_hook):
+    '''
+    Given a re.Match instance of CONSTANT_PATTERN representing a matched constant name to be
+    interpolated, a constants dict, and whether this is for a command hook, lookup the matched
+    constant name within the given constants and return its value.
+
+    If the match is escaped with backslashes, then instead of resolving the variable's value, strip
+    off the backslashing and return the literal value.
+
+    If the variable name isn't found in the given constants, then return the literal value.
+    '''
+    name = match.group('name')
+
+    # The would-be variable is escaped, so strip off the escaping and return the result without
+    # resolving the name—unless this is for a command hook, in which case just return the literal
+    # string. That way, subsequent variable interpolation will still see the string as escaped
+    # instead of trying to interpolate it.
+    if match.group('left_escape') and match.group('right_escape'):
+        if command_hook:
+            return match.group(0)
+
+        return '{' + name + '}'
+
+    value = constants.get(name)
+
+    # The matched variable is in the constants, so return its value. And if this is for a command
+    # hook, then shell escape the value so as to prevent shell injection attacks.
+    if value is not None:
+        return shlex.quote(str(value)) if command_hook else str(value)
+
+    # The matched variable name isn't in the constants. Return the whole string unaltered.
+    return match.group(0)
+
+
+def apply_constants(value, constants, command_hook=False):
     '''
     Given a configuration value (bool, dict, int, list, or string) and a dict of named constants,
     replace any configuration string values of the form "{constant}" (or containing it) with the
@@ -37,27 +78,25 @@ def apply_constants(value, constants, shell_escape=False):
         return value
 
     if isinstance(value, str):
-        for constant_name, constant_value in constants.items():
-            value = value.replace(
-                '{' + constant_name + '}',
-                shlex.quote(str(constant_value)) if shell_escape else str(constant_value),
-            )
-
         # Support constants within non-string scalars by coercing the value to its appropriate type.
-        value = coerce_scalar(value)
+        value = coerce_scalar(
+            CONSTANT_PATTERN.sub(
+                functools.partial(resolve_constant, constants=constants, command_hook=command_hook),
+                value,
+            )
+        )
     elif isinstance(value, list):
         for index, list_value in enumerate(value):
-            value[index] = apply_constants(list_value, constants, shell_escape)
+            value[index] = apply_constants(list_value, constants, command_hook)
     elif isinstance(value, dict):
         for option_name, option_value in value.items():
             value[option_name] = apply_constants(
                 option_value,
                 constants,
-                shell_escape=(
+                command_hook=(
-                    shell_escape
+                    command_hook
-                    or option_name.startswith('before_')
+                    or option_name.startswith(('before_', 'after_'))
-                    or option_name.startswith('after_')
+                    or option_name in {'on_error', 'run'}
-                    or option_name == 'on_error'
                 ),
             )
 

borgmatic/config/environment.py

@@ -2,7 +2,7 @@ import os
 import re
 
 VARIABLE_PATTERN = re.compile(
-    r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
+    r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})',
 )