Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1014

.drone.yml

@@ -1,64 +0,0 @@
----
-kind: pipeline
-name: python-3-8-alpine-3-13
-
-services:
-  - name: postgresql
-    image: docker.io/postgres:13.1-alpine
-    environment:
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: test
-  - name: mysql
-    image: docker.io/mariadb:10.5
-    environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-  - name: mongodb
-    image: docker.io/mongo:5.0.5
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: root
-      MONGO_INITDB_ROOT_PASSWORD: test
-
-clone:
-  skip_verify: true
-
-steps:
-- name: build
-  image: docker.io/alpine:3.13
-  environment:
-    TEST_CONTAINER: true
-  pull: always
-  commands:
-    - scripts/run-full-tests
----
-kind: pipeline
-name: documentation
-type: exec
-
-platform:
-  os: linux
-  arch: amd64
-
-clone:
-  skip_verify: true
-
-steps:
-- name: build
-  environment:
-    USERNAME:
-      from_secret: docker_username
-    PASSWORD:
-      from_secret: docker_password
-    IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
-  commands:
-    - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
-    - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
-    - podman push "$IMAGE_NAME"
-
-trigger:
-  repo:
-    - borgmatic-collective/borgmatic
-  branch:
-    - main
-  event:
-    - push

.flake8

@@ -1 +0,0 @@
-select = Q0

.gitea/issue_template.md

@@ -1,35 +0,0 @@
-#### What I'm trying to do and why
-
-#### Steps to reproduce (if a bug)
-
-Include (sanitized) borgmatic configuration files if applicable.
-
-#### Actual behavior (if a bug)
-
-Include (sanitized) `--verbosity 2` output if applicable.
-
-#### Expected behavior (if a bug)
-
-#### Other notes / implementation ideas
-
-#### Environment
-
-**borgmatic version:** [version here]
-
-Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
-
-**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
-
-**Borg version:** [version here]
-
-Use `sudo borg --version`
-
-**Python version:** [version here]
-
-Use `python3 --version`
-
-**Database version (if applicable):** [version here]
-
-Use `psql --version` or `mysql --version` on client and server.
-
-**operating system and version:** [OS here]

.gitea/issue_template/bug_template.yaml

@@ -0,0 +1,77 @@
+name: "Bug or question/support"
+about: "For filing a bug or getting support"
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: What I'm trying to do and why
+    validations:
+      required: true
+  - type: textarea
+    id: repro_steps
+    attributes:
+      label: Steps to reproduce
+      description: Include (sanitized) borgmatic configuration files if applicable.
+    validations:
+      required: false
+  - type: textarea
+    id: actual_behavior
+    attributes:
+      label: Actual behavior
+      description: Include (sanitized) `--verbosity 2` output if applicable.
+    validations:
+      required: false
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: Expected behavior
+    validations:
+      required: false
+  - type: textarea
+    id: notes
+    attributes:
+      label: Other notes / implementation ideas
+    validations:
+      required: false
+  - type: input
+    id: borgmatic_version
+    attributes:
+      label: borgmatic version
+      description: Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+    validations:
+      required: false
+  - type: input
+    id: borgmatic_install_method
+    attributes:
+      label: borgmatic installation method
+      description: e.g., pip install, Debian package, container, etc.
+    validations:
+      required: false
+  - type: input
+    id: borg_version
+    attributes:
+      label: Borg version
+      description: Use `sudo borg --version`
+    validations:
+      required: false
+  - type: input
+    id: python_version
+    attributes:
+      label: Python version
+      description: Use `python3 --version`
+    validations:
+      required: false
+  - type: input
+    id: database_version
+    attributes:
+      label: Database version (if applicable)
+      description: Use `psql --version` / `mysql --version` / `mongodump --version` / `sqlite3 --version`
+    validations:
+      required: false
+  - type: input
+    id: operating_system_version
+    attributes:
+      label: Operating system and version
+      description: On Linux, use `cat /etc/os-release`
+    validations:
+      required: false

.gitea/issue_template/config.yaml

@@ -0,0 +1 @@
+blank_issues_enabled: true

.gitea/issue_template/feature_template.yaml

@@ -0,0 +1,15 @@
+name: "Feature"
+about: "For filing a feature request or idea"
+body:
+  - type: textarea
+    id: request
+    attributes:
+      label: What I'd like to do and why
+    validations:
+      required: true
+  - type: textarea
+    id: notes
+    attributes:
+      label: Other notes / implementation ideas
+    validations:
+      required: false

.gitea/workflows/build.yaml

@@ -0,0 +1,30 @@
+name: build
+run-name: ${{ gitea.actor }} is building
+on:
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: host
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: scripts/run-end-to-end-tests
+
+  docs:
+    needs: [test]
+    runs-on: host
+    env:
+      IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
+        env:
+          USERNAME: "${{ secrets.REGISTRY_USERNAME }}"
+          PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
+      - run: podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
+      - run: podman push "$IMAGE_NAME"
+      - run: scripts/export-docs-from-image
+      - run: curl --user "${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}" --upload-file borgmatic-docs.tar.gz https://projects.torsion.org/api/packages/borgmatic-collective/generic/borgmatic-docs/$(head --lines=1 NEWS)/borgmatic-docs.tar.gz

NEWS

@@ -1,3 +1,467 @@
+1.9.13
+ * #975: Add a "compression" option to the PostgreSQL database hook.
+ * #1001: Fix a ZFS error during snapshot cleanup.
+ * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes.
+ * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than
+   using an environment variable.
+ * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using
+   "--password" on the command-line!
+ * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit.
+ * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification.
+ * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption
+   between client and server.
+
+1.9.12
+ * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will
+   work with Python 3.9, 3.10, and 3.11 again.
+
+1.9.11
+ * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #996: Fix the "create" action to omit the repository label prefix from Borg's output when
+   databases are enabled.
+ * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure
+   than using an environment variable.
+ * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file
+   detection.
+ * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from
+   a borgmatic configuration option (e.g. "source_directories")—not from other hooks within
+   borgmatic.
+ * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical
+   volumes.
+ * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property.
+ * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
+   source directories.
+
+1.9.10
+ * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic
+   configuration files. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #987: Fix a "list" action error when the "encryption_passcommand" option is set.
+ * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer
+   "encryption_passphrase" even if it's an empty value.
+ * #988: With the "max_duration" option or the "--max-duration" flag, run the archives and
+   repository checks separately so they don't interfere with one another. Previously, borgmatic
+   refused to run checks in this situation.
+ * #989: Fix the log message code to avoid using Python 3.10+ logging features. Now borgmatic will
+   work with Python 3.9 again.
+ * Capture and delay any log records produced before logging is fully configured, so early log
+   records don't get lost.
+ * Add support for Python 3.13.
+
+1.9.9
+ * #635: Log the repository path or label on every relevant log message, not just some logs.
+ * #961: When the "encryption_passcommand" option is set, call the command once from borgmatic to
+   collect the encryption passphrase and then pass it to Borg multiple times. See the documentation
+   for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #981: Fix a "spot" check file count delta error.
+ * #982: Fix for borgmatic "exclude_patterns" and "exclude_from" recursing into excluded
+   subdirectories.
+ * #983: Fix the Btrfs hook to support subvolumes with names like "@home" different from their
+   mount points.
+ * #985: Change the default value for the "--original-hostname" flag from "localhost" to no host
+   specified. This way, the "restore" action works without a hostname if there's a single matching
+   database dump.
+
+1.9.8
+ * #979: Fix root patterns so they don't have an invalid "sh:" prefix before getting passed to Borg.
+ * Expand the recent contributors documentation section to include ticket submitters—not just code
+   contributors—because there are multiple ways to contribute to the project! See:
+   https://torsion.org/borgmatic/#recent-contributors
+
+1.9.7
+ * #855: Add a Sentry monitoring hook. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook
+ * #968: Fix for a "spot" check error when a filename in the most recent archive contains a newline.
+ * #970: Fix for an error when there's a blank line in the configured patterns or excludes.
+ * #971: Fix for "exclude_from" files being completely ignored.
+ * #977: Fix for "exclude_patterns" and "exclude_from" not supporting explicit pattern styles (e.g.,
+   "sh:" or "re:").
+
+1.9.6
+ * #959: Fix an error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
+   source directories.
+ * #960: Fix for archives storing relative source directory paths such that they contain the working
+   directory.
+ * #960: Fix the "spot" check to support relative source directory paths.
+ * #962: For the ZFS, Btrfs, and LVM hooks, perform path rewriting for excludes and patterns in
+   addition to the existing source directories rewriting.
+ * #962: Under the hood, merge all configured source directories, excludes, and patterns into a
+   unified temporary patterns file for passing to Borg. The borgmatic configuration options remain
+   unchanged.
+ * #962: For the LVM hook, add support for nested logical volumes.
+ * #965: Fix a borgmatic runtime directory error when running the "spot" check with a database hook
+   enabled.
+ * #969: Fix the "restore" action to work on database dumps without a port when a default port is
+   present in configuration.
+ * Fix the "spot" check to no longer consider pipe files within an archive for file comparisons.
+ * Fix the "spot" check to have a nicer error when there are no source paths to compare.
+ * Fix auto-excluding of special files (when databases are configured) to support relative source
+   directory paths.
+ * Drop support for Python 3.8, which has been end-of-lifed.
+
+1.9.5
+ * #418: Backup and restore databases that have the same name but with different ports, hostnames,
+   or hooks.
+ * #947: To avoid a hang in the database hooks, error and exit when the borgmatic runtime
+   directory overlaps with the configured excludes.
+ * #954: Fix a findmnt command error in the Btrfs hook by switching to parsing JSON output.
+ * #956: Fix the printing of a color reset code even when color is disabled.
+ * #958: Drop colorama as a library dependency.
+ * When the ZFS, Btrfs, or LVM hooks aren't configured, don't try to cleanup snapshots for them.
+
+1.9.4
+ * #80 (beta): Add an LVM hook for snapshotting and backing up LVM logical volumes. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
+ * #251 (beta): Add a Btrfs hook for snapshotting and backing up Btrfs subvolumes. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
+ * #926: Fix a library error when running within a PyInstaller bundle.
+ * #950: Fix a snapshot unmount error in the ZFS hook when using nested datasets.
+ * Update the ZFS hook to discover and snapshot ZFS datasets even if they are parent/grandparent
+   directories of your source directories.
+ * Reorganize data source and monitoring hooks to make developing new hooks easier.
+
+1.9.3
+ * #261 (beta): Add a ZFS hook for snapshotting and backing up ZFS datasets. See the documentation
+   for more information: https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
+ * Remove any temporary copies of the manifest file created in support of the "bootstrap" action.
+ * Deprecate the "store_config_files" option at the global scope and move it under the "bootstrap"
+   hook. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
+ * Require the runtime directory to be an absolute path.
+ * Add a "--deleted" flag to the "repo-list" action for listing deleted archives that haven't
+   yet been compacted (Borg 2 only).
+ * Promote the "spot" check from a beta feature to stable.
+
+1.9.2
+ * #441: Apply the "umask" option to all relevant actions, not just some of them.
+ * #722: Remove the restriction that the "extract" and "mount" actions must match a single
+   repository. Now they work more like other actions, where each repository is applied in turn.
+ * #932: Fix the missing build backend setting in pyproject.toml to allow Fedora builds.
+ * #934: Update the logic that probes for the borgmatic streaming database dump, bootstrap
+   metadata, and check state directories to support more platforms and use cases. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory
+ * #934: Add the "RuntimeDirectory" and "StateDirectory" options to the sample systemd service
+   file to support the new runtime and state directory logic.
+ * #939: Fix borgmatic ignoring the "BORG_RELOCATED_REPO_ACCESS_IS_OK" and
+   "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK" environment variables.
+ * Add a Pushover monitoring hook. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook
+
+1.9.1
+ * #928: Fix the user runtime directory location on macOS (and possibly Cygwin).
+ * #930: Fix an error with the sample systemd service when no credentials are configured.
+ * #931: Fix an error when implicitly upgrading the check state directory from ~/.borgmatic to
+   ~/.local/state/borgmatic across filesystems.
+
+1.9.0
+ * #609: Fix the glob expansion of "source_directories" values to respect the "working_directory"
+   option.
+ * #609: BREAKING: Apply the "working_directory" option to all actions, not just "create". This
+   includes repository paths, destination paths, mount points, etc.
+ * #562: Deprecate the "borgmatic_source_directory" option in favor of "user_runtime_directory"
+   and "user_state_directory".
+ * #562: BREAKING: Move the default borgmatic streaming database dump and bootstrap metadata
+   directory from ~/.borgmatic to /run/user/$UID/borgmatic, which is more XDG-compliant. You can
+   override this location with the new "user_runtime_directory" option. Existing archives with
+   database dumps at the old location are still restorable. 
+ * #562, #638: Move the default check state directory from ~/.borgmatic to 
+   ~/.local/state/borgmatic. This is more XDG-compliant and also prevents these state files from
+   getting backed up (unless you explicitly include them). You can override this location with the
+   new "user_state_directory" option. After the first time you run the "check" action with borgmatic
+   1.9.0, you can safely delete the ~/.borgmatic directory.
+ * #838: BREAKING: With Borg 1.4+, store database dumps and bootstrap metadata in a "/borgmatic"
+   directory within a backup archive, so the path doesn't depend on the current user. This means
+   that you can now backup as one user and restore or bootstrap as another user, among other use
+   cases.
+ * #902: Add loading of encrypted systemd credentials. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/#using-systemd-service-credentials
+ * #911: Add a "key change-passphrase" action to change the passphrase protecting a repository key.
+ * #914: Fix a confusing apparent hang when when the repository location changes, and instead
+   show a helpful error message.
+ * #915: BREAKING: Rename repository actions like "rcreate" to more explicit names like
+   "repo-create" for compatibility with recent changes in Borg 2.0.0b10.
+ * #918: BREAKING: When databases are configured, don't auto-enable the "one_file_system" option,
+   as existing auto-excludes of special files should be sufficient to prevent Borg from hanging on
+   them. But if this change causes problems for you, you can always enable "one_file_system"
+   explicitly.
+ * #919: Clarify the command-line help for the "--config" flag.
+ * #919: Document a policy for versioning and breaking changes:
+   https://torsion.org/borgmatic/docs/how-to/upgrade/#versioning-and-breaking-changes
+ * #921: BREAKING: Change soft failure command hooks to skip only the current repository rather than
+   all repositories in the configuration file.
+ * #922: Replace setup.py (Python packaging metadata) with the more modern pyproject.toml.
+ * When using Borg 2, default the "archive_name_format" option to just "{hostname}", as Borg 2 does
+   not require unique archive names; identical archive names form a common "series" that can be
+   targeted together. See the Borg 2 documentation for more information:
+   https://borgbackup.readthedocs.io/en/2.0.0b13/changes.html#borg-1-2-x-1-4-x-to-borg-2-0
+ * Add support for Borg 2's "rclone:" repository URLs, so you can backup to 70+ cloud storage
+   services whether or not they support Borg explicitly.
+ * Add support for Borg 2's "sftp://" repository URLs.
+ * Update the "--match-archives" and "--archive" flags to support Borg 2 series names or archive
+   hashes.
+ * Add a "--match-archives" flag to the "prune" action.
+ * Add "--local-path" and "--remote-path" flags to the "config bootstrap" action for setting the
+   Borg executable paths used for bootstrapping.
+ * Add a "--user-runtime-directory" flag to the "config bootstrap" action for helping borgmatic
+   locate the bootstrap metadata stored in an archive.
+ * Add a Zabbix monitoring hook. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook
+ * Add a tarball of borgmatic's HTML documentation to the packages on the project page.
+
+1.8.14
+ * #896: Fix an error in borgmatic rcreate/init on an empty repository directory with Borg 1.4.
+ * #898: Add glob ("*") support to the "--repository" flag. Just quote any values containing
+   globs so your shell doesn't interpret them.
+ * #899: Fix for a "bad character" Borg error in which the "spot" check fed Borg an invalid pattern.
+ * #900: Fix for a potential traceback (TypeError) during the handling of another error.
+ * #904: Clarify the configuration reference about the "spot" check options:
+   https://torsion.org/borgmatic/docs/reference/configuration/
+ * #905: Fix the "source_directories_must_exist" option to work with relative "source_directories"
+   paths when a "working_directory" is set.
+ * #906: Add documentation details for how to run custom database dump commands using binaries from
+   running containers:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+ * Fix a regression in which the "color" option had no effect.
+ * Add a recent contributors section to the documentation, because credit where credit's due! See:
+   https://torsion.org/borgmatic/#recent-contributors
+
+1.8.13
+ * #298: Add "delete" and "rdelete" actions to delete archives or entire repositories.
+ * #785: Add an "only_run_on" option to consistency checks so you can limit a check to running on
+   particular days of the week. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-days
+ * #885: Add an Uptime Kuma monitoring hook. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook
+ * #886: Fix a PagerDuty hook traceback with Python < 3.10.
+ * #889: Fix the Healthchecks ping body size limit, restoring it to the documented 100,000 bytes.
+
+1.8.12
+ * #817: Add a "--max-duration" flag to the "check" action and a "max_duration" option to the
+   repository check configuration. This tells Borg to interrupt a repository check after a certain
+   duration.
+ * #860: Fix interaction between environment variable interpolation in constants and shell escaping.
+ * #863: When color output is disabled (explicitly or implicitly), don't prefix each log line with
+   the log level.
+ * #865: Add an "upload_buffer_size" option to set the size of the upload buffer used in "create"
+   action.
+ * #866: Fix "Argument list too long" error in the "spot" check when checking hundreds of thousands
+   of files at once.
+ * #874: Add the configured repository label as "repository_label" to the interpolated variables
+   passed to before/after command hooks.
+ * #881: Fix "Unrecognized argument" error when the same value is used with different command-line
+   flags.
+ * In the "spot" check, don't try to hash symlinked directories.
+
+1.8.11
+ * #815: Add optional Healthchecks auto-provisioning via "create_slug" option.
+ * #851: Fix lack of file extraction when using "extract --strip-components all" on a path with a
+   leading slash.
+ * #854: Fix a traceback when the "data" consistency check is used.
+ * #857: Fix a traceback with "check --only spot" when the "spot" check is unconfigured.
+
+1.8.10
+ * #656 (beta): Add a "spot" consistency check that compares file counts and contents between your
+   source files and the latest archive, ensuring they fall within configured tolerances. This can
+   catch problems like incorrect excludes, inadvertent deletes, files changed by malware, etc. See
+   the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#spot-check
+ * #779: When "--match-archives *" is used with "check" action, don't skip Borg's orphaned objects
+   check.
+ * #842: When a command hook exits with a soft failure, ping the log and finish states for any
+   configured monitoring hooks.
+ * #843: Add documentation link to Loki dashboard for borgmatic:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
+ * #847: Fix "--json" error when Borg includes non-JSON warnings in JSON output.
+ * #848: SECURITY: Mask the password when logging a MongoDB dump or restore command.
+ * Fix handling of the NO_COLOR environment variable to ignore an empty value.
+ * Add documentation about backing up containerized databases by configuring borgmatic to exec into
+   a container to run a dump command:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+
+1.8.9
+ * #311: Add custom dump/restore command options for MySQL and MariaDB.
+ * #811: Add an "access_token" option to the ntfy monitoring hook for authenticating
+   without username/password.
+ * #827: When the "--json" flag is given, suppress console escape codes so as not to
+   interfere with JSON output.
+ * #829: Fix "--override" values containing deprecated section headers not actually overriding
+   configuration options under deprecated section headers.
+ * #835: Add support for the NO_COLOR environment variable. See the documentation for more
+   information:
+   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#colored-output
+ * #839: Add log sending for the Apprise logging hook, enabled by default. See the documentation for
+   more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook
+ * #839: Document a potentially breaking shell quoting edge case within error hooks:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks
+ * #840: When running the "rcreate" action and the repository already exists but with a different
+   encryption mode than requested, error.
+ * Switch from Drone to Gitea Actions for continuous integration.
+ * Rename scripts/run-end-to-end-dev-tests to scripts/run-end-to-end-tests and use it in both dev
+   and CI for better dev-CI parity.
+ * Clarify documentation about restoring a database: borgmatic does not create the database upon
+   restore.
+
+1.8.8
+ * #370: For the PostgreSQL hook, pass the "PGSSLMODE" environment variable through to Borg when the
+   database's configuration omits the "ssl_mode" option.
+ * #818: Allow the "--repository" flag to match across multiple configuration files.
+ * #820: Fix broken repository detection in the "rcreate" action with Borg 1.4. The issue did not
+   occur with other versions of Borg.
+ * #822: Fix broken escaping logic in the PostgreSQL hook's "pg_dump_command" option.
+ * SECURITY: Prevent additional shell injection attacks within the PostgreSQL hook.
+
+1.8.7
+ * #736: Store included configuration files within each backup archive in support of the "config
+   bootstrap" action. Previously, only top-level configuration files were stored.
+ * #798: Elevate specific Borg warnings to errors or squash errors to
+ * warnings. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/customize-warnings-and-errors/
+ * #810: SECURITY: Prevent shell injection attacks within the PostgreSQL hook, the MongoDB hook, the
+   SQLite hook, the "borgmatic borg" action, and command hook variable/constant interpolation.
+ * #814: Fix a traceback when providing an invalid "--override" value for a list option.
+
+1.8.6
+ * #767: Add an "--ssh-command" flag to the "config bootstrap" action for setting a custom SSH
+   command, as no configuration is available (including the "ssh_command" option) until
+   bootstrapping completes.
+ * #794: Fix a traceback when the "repositories" option contains both strings and key/value pairs.
+ * #800: Add configured repository labels to the JSON output for all actions.
+ * #802: The "check --force" flag now runs checks even if "check" is in "skip_actions".
+ * #804: Validate the configured action names in the "skip_actions" option.
+ * #807: Stream SQLite databases directly to Borg instead of dumping to an intermediate file.
+ * When logging commands that borgmatic executes, log the environment variables that
+   borgmatic sets for those commands. (But don't log their values, since they often contain
+   passwords.)
+
+1.8.5
+ * #701: Add a "skip_actions" option to skip running particular actions, handy for append-only or
+   checkless configurations. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#skipping-actions
+ * #701: Deprecate the "disabled" value for the "checks" option in favor of the new "skip_actions"
+   option.
+ * #745: Constants now apply to included configuration, not just the file doing the includes. As a
+   side effect of this change, constants no longer apply to option names and only substitute into
+   configuration values.
+ * #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check,
+   overriding the existing "archive_name_format" and "match_archives" options in configuration.
+ * #779: Only parse "--override" values as complex data types when they're for options of those
+   types.
+ * #782: Fix environment variable interpolation within configured repository paths.
+ * #782: Add configuration constant overriding via the existing "--override" flag.
+ * #783: Upgrade ruamel.yaml dependency to support version 0.18.x.
+ * #784: Drop support for Python 3.7, which has been end-of-lifed.
+
+1.8.4
+ * #715: Add a monitoring hook for sending backup status to a variety of monitoring services via the
+   Apprise library. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook
+ * #748: When an archive filter causes no matching archives for the "rlist" or "info"
+   actions, warn the user and suggest how to remove the filter.
+ * #768: Fix a traceback when an invalid command-line flag or action is used.
+ * #771: Fix normalization of deprecated sections ("location:", "storage:", "hooks:", etc.) to
+   support empty sections without erroring.
+ * #774: Disallow the "--dry-run" flag with the "borg" action, as borgmatic can't guarantee the Borg
+   command won't have side effects.
+
+1.8.3
+ * #665: BREAKING: Simplify logging logic as follows: Syslog verbosity is now disabled by
+   default, but setting the "--syslog-verbosity" flag enables it regardless of whether you're at an
+   interactive console. Additionally, "--log-file-verbosity" and "--monitoring-verbosity" now
+   default to 1 (info about steps borgmatic is taking) instead of 0. And both syslog logging and
+   file logging can be enabled simultaneously.
+ * #743: Add a monitoring hook for sending backup status and logs to Grafana Loki. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook
+ * #753: When "archive_name_format" is not set, filter archives using the default archive name
+   format.
+ * #754: Fix error handling to log command output as one record per line instead of truncating
+   too-long output and swallowing the end of some Borg error messages.
+ * #757: Update documentation so "sudo borgmatic" works for pipx borgmatic installations.
+ * #761: Fix for borgmatic not stopping Borg immediately when the user presses ctrl-C.
+ * Update documentation to recommend installing/upgrading borgmatic with pipx instead of pip. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#installation
+   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-borgmatic
+
+1.8.2
+ * #345: Add "key export" action to export a copy of the repository key for safekeeping in case
+   the original goes missing or gets damaged.
+ * #727: Add a MariaDB database hook that uses native MariaDB commands instead of the deprecated
+   MySQL ones. Be aware though that any existing backups made with the "mysql_databases:" hook are
+   only restorable with a "mysql_databases:" configuration.
+ * #738: Fix for potential data loss (data not getting restored) in which the database "restore"
+   action didn't actually restore anything and indicated success anyway.
+ * Remove the deprecated use of the MongoDB hook's "--db" flag for database restoration.
+ * Add source code reference documentation for getting oriented with the borgmatic code as a
+   developer: https://torsion.org/borgmatic/docs/reference/source-code/
+
+1.8.1
+ * #326: Add documentation for restoring a database to an alternate host:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-to-an-alternate-host
+ * #697: Add documentation for "bootstrap" action:
+   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
+ * #725: Add "store_config_files" option for disabling the automatic backup of configuration files
+   used by the "config bootstrap" action.
+ * #728: Fix for "prune" action error when using the "keep_exclude_tags" option.
+ * #730: Fix for Borg's interactive prompt on the "check --repair" action automatically getting
+   answered "NO" even when the "check_i_know_what_i_am_doing" option isn't set.
+ * #732: Include multiple configuration files with a single "!include". See the documentation for
+   more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#multiple-merge-includes
+ * #734: Omit "--glob-archives" or "--match-archives" Borg flag when its value would be "*" (meaning
+   all archives).
+
+1.8.0
+ * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting
+   repository/archive into the resulting Borg command-line, pass repository to Borg via an
+   environment variable and make archive available for explicit use in your commands. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/
+ * #719: Fix an error when running "borg key export" through borgmatic.
+ * #720: Fix an error when dumping a database and the "exclude_nodump" option is set.
+ * #724: Add "check_i_know_what_i_am_doing" option to bypass Borg confirmation prompt when running
+   "check --repair".
+ * When merging two configuration files, error gracefully if the two files do not adhere to the same
+   format.
+ * #721: Remove configuration sections ("location:", "storage:", "hooks:", etc.), while still
+   keeping deprecated support for them. Now, all options are at the same level, and you don't need
+   to worry about commenting/uncommenting section headers when you change an option (if you remove
+   your sections first).
+ * #721: BREAKING: The retention prefix and the consistency prefix can no longer have different
+   values (unless one is not set).
+ * #721: BREAKING: The storage umask and the hooks umask can no longer have different values (unless
+   one is not set).
+ * BREAKING: Flags like "--config" that previously took multiple values now need to be given once
+   per value, e.g. "--config first.yaml --config second.yaml" instead of "--config first.yaml
+   second.yaml". This prevents argument parsing errors on ambiguous commands.
+ * BREAKING: Remove the deprecated (and silently ignored) "--successful" flag on the "list" action,
+   as newer versions of Borg list successful (non-checkpoint) archives by default.
+ * All deprecated configuration option values now generate warning logs.
+ * Remove the deprecated (and non-functional) "--excludes" flag in favor of excludes within
+   configuration.
+ * Fix an error when logging too-long command output during error handling. Now, long command output
+   is truncated before logging.
+
+1.7.15
+ * #326: Add configuration options and command-line flags for backing up a database from one
+   location while restoring it somewhere else.
+ * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors.
+ * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style
+   configuration.
+ * #529: Deprecate generate-borgmatic-config in favor of new "config generate" action.
+ * #529: Deprecate validate-borgmatic-config in favor of new "config validate" action.
+ * #697, #712, #716: Extract borgmatic configuration from backup via new "config bootstrap"
+   action—even when borgmatic has no configuration yet!
+ * #669: Add sample systemd user service for running borgmatic as a non-root user.
+ * #711, #713: Fix an error when "data" check time files are accessed without getting upgraded
+   first.
+
 1.7.14
  * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file,
    or monitoring), so not even errors are shown.

README.md

@@ -16,65 +16,81 @@ The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">ht
 Here's an example configuration file:
 
 ```yaml
-location:
-    # List of source directories to backup.
-    source_directories:
-        - /home
-        - /etc
+# List of source directories to backup.
+source_directories:
+    - /home
+    - /etc
 
-    # Paths of local or remote repositories to backup to.
-    repositories:
-        - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
-          label: borgbase
-        - path: /var/lib/backups/local.borg
-          label: local
+# Paths of local or remote repositories to backup to.
+repositories:
+    - path: ssh://k8pDxu32@k8pDxu32.repo.borgbase.com/./repo
+      label: borgbase
+    - path: /var/lib/backups/local.borg
+      label: local
 
-retention:
-    # Retention policy for how many backups to keep.
-    keep_daily: 7
-    keep_weekly: 4
-    keep_monthly: 6
+# Retention policy for how many backups to keep.
+keep_daily: 7
+keep_weekly: 4
+keep_monthly: 6
 
-consistency:
-    # List of checks to run to validate your backups.
-    checks:
-        - name: repository
-        - name: archives
-          frequency: 2 weeks
+# List of checks to run to validate your backups.
+checks:
+    - name: repository
+    - name: archives
+      frequency: 2 weeks
 
-hooks:
-    # Custom preparation scripts to run.
-    before_backup:
-        - prepare-for-backup.sh
+# Custom preparation scripts to run.
+before_backup:
+    - prepare-for-backup.sh
 
-    # Databases to dump and include in backups.
-    postgresql_databases:
-        - name: users
+# Databases to dump and include in backups.
+postgresql_databases:
+    - name: users
 
-    # Third-party services to notify you if backups aren't happening.
-    healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
+# Third-party services to notify you if backups aren't happening.
+healthchecks:
+    ping_url: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
 ```
 
-Want to see borgmatic in action? Check out the <a
-href="https://asciinema.org/a/203761?autoplay=1" target="_blank">screencast</a>.
-
-<a href="https://asciinema.org/a/203761?autoplay=1" target="_blank"><img src="https://asciinema.org/a/203761.png" width="480"></a>
-
 borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
 ## Integrations
 
-<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+### Data
+
+<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://openzfs.org/"><img src="docs/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://btrfs.readthedocs.io/"><img src="docs/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sourceware.org/lvm2/"><img src="docs/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://rclone.org"><img src="docs/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+
+
+### Monitoring
+
+<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://uptime.kuma.pet/"><img src="docs/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.pushover.net/"><img src="docs/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+
+
+### Credentials
+
+<a href="https://systemd.io/"><img src="docs/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.docker.com/"><img src="docs/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://podman.io/"><img src="docs/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://keepassxc.org/"><img src="docs/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ## Getting started
@@ -91,16 +107,15 @@ reference guides</a>.
 
 Need somewhere to store your encrypted off-site backups? The following hosting
 providers include specific support for Borg/borgmatic—and fund borgmatic
-development and hosting when you use these links to sign up. (These are
-referral links, but without any tracking scripts or cookies.)
+development and hosting when you use these referral links to sign up:
 
 <ul>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+ <li class="referral"><a href="https://hetzner.cloud/?ref=v9dOJ98Ic9I8">Hetzner</a>: A "storage box" that includes support for Borg</li>
 </ul>
 
-Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and
-[Hetzner](https://www.hetzner.com/storage/storage-box) have compatible storage
-offerings, but do not currently fund borgmatic development or hosting.
+Additionally, rsync.net has a compatible storage offering, but does not fund
+borgmatic development or hosting.
 
 ## Support and contributing
 
@@ -121,10 +136,7 @@ issues.
 
 ### Social
 
-Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for
-general Borg and borgmatic discussion and support.
-
-Also follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
+Follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
 
 
 ### Chat
@@ -165,5 +177,10 @@ Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
-<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/main)</a>
+### Recent contributors
 
+Thanks to all borgmatic contributors! There are multiple ways to contribute to
+this project, so the following includes those who have fixed bugs, contributed
+features, *or* filed tickets.
+
+{% include borgmatic/contributors.html %}

borgmatic/actions/borg.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.borg
-import borgmatic.borg.rlist
+import borgmatic.borg.repo_list
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 def run_borg(
     repository,
-    storage,
+    config,
     local_borg_version,
     borg_arguments,
     global_arguments,
@@ -22,13 +22,11 @@ def run_borg(
     if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, borg_arguments.repository
     ):
-        logger.info(
-            f'{repository.get("label", repository["path"])}: Running arbitrary Borg command'
-        )
-        archive_name = borgmatic.borg.rlist.resolve_archive_name(
+        logger.info('Running arbitrary Borg command')
+        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
             borg_arguments.archive,
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path,
@@ -36,7 +34,7 @@ def run_borg(
         )
         borgmatic.borg.borg.run_arbitrary_borg(
             repository['path'],
-            storage,
+            config,
             local_borg_version,
             options=borg_arguments.options,
             archive=archive_name,

borgmatic/actions/break_lock.py

@@ -8,7 +8,7 @@ logger = logging.getLogger(__name__)
 
 def run_break_lock(
     repository,
-    storage,
+    config,
     local_borg_version,
     break_lock_arguments,
     global_arguments,
@@ -21,12 +21,10 @@ def run_break_lock(
     if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, break_lock_arguments.repository
     ):
-        logger.info(
-            f'{repository.get("label", repository["path"])}: Breaking repository and cache locks'
-        )
+        logger.info('Breaking repository and cache locks')
         borgmatic.borg.break_lock.break_lock(
             repository['path'],
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path=local_path,

borgmatic/actions/change_passphrase.py

@@ -0,0 +1,36 @@
+import logging
+
+import borgmatic.borg.change_passphrase
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_change_passphrase(
+    repository,
+    config,
+    local_borg_version,
+    change_passphrase_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "key change-passphrase" action for the given repository.
+    '''
+    if (
+        change_passphrase_arguments.repository is None
+        or borgmatic.config.validate.repositories_match(
+            repository, change_passphrase_arguments.repository
+        )
+    ):
+        logger.info('Changing repository passphrase')
+        borgmatic.borg.change_passphrase.change_passphrase(
+            repository['path'],
+            config,
+            local_borg_version,
+            change_passphrase_arguments,
+            global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+        )

borgmatic/actions/check.py

@@ -1,19 +1,687 @@
+import calendar
+import datetime
+import hashlib
+import itertools
 import logging
+import os
+import pathlib
+import random
+import shutil
 
+import borgmatic.actions.create
 import borgmatic.borg.check
+import borgmatic.borg.create
+import borgmatic.borg.environment
+import borgmatic.borg.extract
+import borgmatic.borg.list
+import borgmatic.borg.repo_list
+import borgmatic.borg.state
+import borgmatic.config.paths
 import borgmatic.config.validate
+import borgmatic.execute
 import borgmatic.hooks.command
 
+DEFAULT_CHECKS = (
+    {'name': 'repository', 'frequency': '1 month'},
+    {'name': 'archives', 'frequency': '1 month'},
+)
+
+
 logger = logging.getLogger(__name__)
 
 
+def parse_checks(config, only_checks=None):
+    '''
+    Given a configuration dict with a "checks" sequence of dicts and an optional list of override
+    checks, return a tuple of named checks to run.
+
+    For example, given a config of:
+
+        {'checks': ({'name': 'repository'}, {'name': 'archives'})}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If a checks value
+    has a name of "disabled", return an empty tuple, meaning that no checks should be run.
+    '''
+    checks = only_checks or tuple(
+        check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS)
+    )
+    checks = tuple(check.lower() for check in checks)
+
+    if 'disabled' in checks:
+        logger.warning(
+            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
+        )
+        if len(checks) > 1:
+            logger.warning(
+                'Multiple checks are configured, but one of them is "disabled"; not running any checks'
+            )
+        return ()
+
+    return checks
+
+
+def parse_frequency(frequency):
+    '''
+    Given a frequency string with a number and a unit of time, return a corresponding
+    datetime.timedelta instance or None if the frequency is None or "always".
+
+    For instance, given "3 weeks", return datetime.timedelta(weeks=3)
+
+    Raise ValueError if the given frequency cannot be parsed.
+    '''
+    if not frequency:
+        return None
+
+    frequency = frequency.strip().lower()
+
+    if frequency == 'always':
+        return None
+
+    try:
+        number, time_unit = frequency.split(' ')
+        number = int(number)
+    except ValueError:
+        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
+
+    if not time_unit.endswith('s'):
+        time_unit += 's'
+
+    if time_unit == 'months':
+        number *= 30
+        time_unit = 'days'
+    elif time_unit == 'years':
+        number *= 365
+        time_unit = 'days'
+
+    try:
+        return datetime.timedelta(**{time_unit: number})
+    except TypeError:
+        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
+
+
+WEEKDAY_DAYS = calendar.day_name[0:5]
+WEEKEND_DAYS = calendar.day_name[5:7]
+
+
+def filter_checks_on_frequency(
+    config,
+    borg_repository_id,
+    checks,
+    force,
+    archives_check_id=None,
+    datetime_now=datetime.datetime.now,
+):
+    '''
+    Given a configuration dict with a "checks" sequence of dicts, a Borg repository ID, a sequence
+    of checks, whether to force checks to run, and an ID for the archives check potentially being
+    run (if any), filter down those checks based on the configured "frequency" for each check as
+    compared to its check time file.
+
+    In other words, a check whose check time file's timestamp is too new (based on the configured
+    frequency) will get cut from the returned sequence of checks. Example:
+
+    config = {
+        'checks': [
+            {
+                'name': 'archives',
+                'frequency': '2 weeks',
+            },
+        ]
+    }
+
+    When this function is called with that config and "archives" in checks, "archives" will get
+    filtered out of the returned result if its check time file is newer than 2 weeks old, indicating
+    that it's not yet time to run that check again.
+
+    Raise ValueError if a frequency cannot be parsed.
+    '''
+    if not checks:
+        return checks
+
+    filtered_checks = list(checks)
+
+    if force:
+        return tuple(filtered_checks)
+
+    for check_config in config.get('checks', DEFAULT_CHECKS):
+        check = check_config['name']
+        if checks and check not in checks:
+            continue
+
+        only_run_on = check_config.get('only_run_on')
+        if only_run_on:
+            # Use a dict instead of a set to preserve ordering.
+            days = dict.fromkeys(only_run_on)
+
+            if 'weekday' in days:
+                days = {
+                    **dict.fromkeys(day for day in days if day != 'weekday'),
+                    **dict.fromkeys(WEEKDAY_DAYS),
+                }
+            if 'weekend' in days:
+                days = {
+                    **dict.fromkeys(day for day in days if day != 'weekend'),
+                    **dict.fromkeys(WEEKEND_DAYS),
+                }
+
+            if calendar.day_name[datetime_now().weekday()] not in days:
+                logger.info(
+                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)"
+                )
+                filtered_checks.remove(check)
+                continue
+
+        frequency_delta = parse_frequency(check_config.get('frequency'))
+        if not frequency_delta:
+            continue
+
+        check_time = probe_for_check_time(config, borg_repository_id, check, archives_check_id)
+        if not check_time:
+            continue
+
+        # If we've not yet reached the time when the frequency dictates we're ready for another
+        # check, skip this check.
+        if datetime_now() < check_time + frequency_delta:
+            remaining = check_time + frequency_delta - datetime_now()
+            logger.info(
+                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
+            )
+            filtered_checks.remove(check)
+
+    return tuple(filtered_checks)
+
+
+def make_archives_check_id(archive_filter_flags):
+    '''
+    Given a sequence of flags to filter archives, return a unique hash corresponding to those
+    particular flags. If there are no flags, return None.
+    '''
+    if not archive_filter_flags:
+        return None
+
+    return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
+
+
+def make_check_time_path(config, borg_repository_id, check_type, archives_check_id=None):
+    '''
+    Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
+    "archives", etc.), and a unique hash of the archives filter flags, return a path for recording
+    that check's time (the time of that check last occurring).
+    '''
+    borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
+
+    if check_type in ('archives', 'data'):
+        return os.path.join(
+            borgmatic_state_directory,
+            'checks',
+            borg_repository_id,
+            check_type,
+            archives_check_id if archives_check_id else 'all',
+        )
+
+    return os.path.join(
+        borgmatic_state_directory,
+        'checks',
+        borg_repository_id,
+        check_type,
+    )
+
+
+def write_check_time(path):  # pragma: no cover
+    '''
+    Record a check time of now as the modification time of the given path.
+    '''
+    logger.debug(f'Writing check time at {path}')
+
+    os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
+    pathlib.Path(path, mode=0o600).touch()
+
+
+def read_check_time(path):
+    '''
+    Return the check time based on the modification time of the given path. Return None if the path
+    doesn't exist.
+    '''
+    logger.debug(f'Reading check time from {path}')
+
+    try:
+        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
+    except FileNotFoundError:
+        return None
+
+
+def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
+    '''
+    Given a configuration dict, a Borg repository ID, the name of a check type ("repository",
+    "archives", etc.), and a unique hash of the archives filter flags, return the corresponding
+    check time or None if such a check time does not exist.
+
+    When the check type is "archives" or "data", this function probes two different paths to find
+    the check time, e.g.:
+
+      ~/.borgmatic/checks/1234567890/archives/9876543210
+      ~/.borgmatic/checks/1234567890/archives/all
+
+    ... and returns the maximum modification time of the files found (if any). The first path
+    represents a more specific archives check time (a check on a subset of archives), and the second
+    is a fallback to the last "all" archives check.
+
+    For other check types, this function reads from a single check time path, e.g.:
+
+      ~/.borgmatic/checks/1234567890/repository
+    '''
+    check_times = (
+        read_check_time(group[0])
+        for group in itertools.groupby(
+            (
+                make_check_time_path(config, borg_repository_id, check, archives_check_id),
+                make_check_time_path(config, borg_repository_id, check),
+            )
+        )
+    )
+
+    try:
+        return max(check_time for check_time in check_times if check_time)
+    except ValueError:
+        return None
+
+
+def upgrade_check_times(config, borg_repository_id):
+    '''
+    Given a configuration dict and a Borg repository ID, upgrade any corresponding check times on
+    disk from old-style paths to new-style paths.
+
+    One upgrade performed is moving the checks directory from:
+
+      {borgmatic_source_directory}/checks (e.g., ~/.borgmatic/checks)
+
+    to:
+
+      {borgmatic_state_directory}/checks (e.g. ~/.local/state/borgmatic)
+
+    Another upgrade is renaming an archive or data check path that looks like:
+
+      {borgmatic_state_directory}/checks/1234567890/archives
+
+    to:
+
+      {borgmatic_state_directory}/checks/1234567890/archives/all
+    '''
+    borgmatic_source_checks_path = os.path.join(
+        borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
+    )
+    borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
+    borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
+
+    if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
+        borgmatic_state_checks_path
+    ):
+        logger.debug(
+            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
+        )
+        os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
+        shutil.move(borgmatic_source_checks_path, borgmatic_state_checks_path)
+
+    for check_type in ('archives', 'data'):
+        new_path = make_check_time_path(config, borg_repository_id, check_type, 'all')
+        old_path = os.path.dirname(new_path)
+        temporary_path = f'{old_path}.temp'
+
+        if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
+            continue
+
+        logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
+
+        try:
+            shutil.move(old_path, temporary_path)
+        except FileNotFoundError:
+            pass
+
+        os.mkdir(old_path)
+        shutil.move(temporary_path, new_path)
+
+
+def collect_spot_check_source_paths(
+    repository,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
+    borgmatic_runtime_directory,
+):
+    '''
+    Given a repository configuration dict, a configuration dict, the local Borg version, global
+    arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
+    collect the source paths that Borg would use in an actual create (but only include files).
+    '''
+    stream_processes = any(
+        borgmatic.hooks.dispatch.call_hooks(
+            'use_streaming',
+            config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+        ).values()
+    )
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    (create_flags, create_positional_arguments, pattern_file) = (
+        borgmatic.borg.create.make_base_create_command(
+            dry_run=True,
+            repository_path=repository['path'],
+            config=config,
+            patterns=borgmatic.actions.create.process_patterns(
+                borgmatic.actions.create.collect_patterns(config),
+                working_directory,
+            ),
+            local_borg_version=local_borg_version,
+            global_arguments=global_arguments,
+            borgmatic_runtime_directory=borgmatic_runtime_directory,
+            local_path=local_path,
+            remote_path=remote_path,
+            list_files=True,
+            stream_processes=stream_processes,
+        )
+    )
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    paths_output = borgmatic.execute.execute_command_and_capture_output(
+        create_flags + create_positional_arguments,
+        capture_stderr=True,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )
+
+    paths = tuple(
+        path_line.split(' ', 1)[1]
+        for path_line in paths_output.splitlines()
+        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
+    )
+
+    return tuple(
+        path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
+    )
+
+
+BORG_DIRECTORY_FILE_TYPE = 'd'
+BORG_PIPE_FILE_TYPE = 'p'
+
+
+def collect_spot_check_archive_paths(
+    repository,
+    archive,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
+    borgmatic_runtime_directory,
+):
+    '''
+    Given a repository configuration dict, the name of the latest archive, a configuration dict, the
+    local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
+    remote Borg path, and the borgmatic runtime directory, collect the paths from the given archive
+    (but only include files and symlinks and exclude borgmatic runtime directories).
+
+    These paths do not have a leading slash, as that's how Borg stores them. As a result, we don't
+    know whether they came from absolute or relative source directories.
+    '''
+    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
+
+    return tuple(
+        path
+        for line in borgmatic.borg.list.capture_archive_listing(
+            repository['path'],
+            archive,
+            config,
+            local_borg_version,
+            global_arguments,
+            path_format='{type} {path}{NUL}',  # noqa: FS003
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+        for (file_type, path) in (line.split(' ', 1),)
+        if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
+        if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
+        if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
+        not in pathlib.Path(path).parents
+        if pathlib.Path(borgmatic_runtime_directory.lstrip(os.path.sep))
+        not in pathlib.Path(path).parents
+    )
+
+
+SAMPLE_PATHS_SUBSET_COUNT = 10000
+
+
+def compare_spot_check_hashes(
+    repository,
+    archive,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
+    source_paths,
+):
+    '''
+    Given a repository configuration dict, the name of the latest archive, a configuration dict, the
+    local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
+    remote Borg path, and spot check source paths, compare the hashes for a sampling of the source
+    paths with hashes from corresponding paths in the given archive. Return a sequence of the paths
+    that fail that hash comparison.
+    '''
+    # Based on the configured sample percentage, come up with a list of random sample files from the
+    # source directories.
+    spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
+    sample_count = max(
+        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
+    )
+    source_sample_paths = tuple(random.sample(source_paths, sample_count))
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+    existing_source_sample_paths = {
+        source_path
+        for source_path in source_sample_paths
+        if os.path.exists(os.path.join(working_directory or '', source_path))
+    }
+    logger.debug(
+        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
+    )
+
+    source_sample_paths_iterator = iter(source_sample_paths)
+    source_hashes = {}
+    archive_hashes = {}
+
+    # Only hash a few thousand files at a time (a subset of the total paths) to avoid an "Argument
+    # list too long" shell error.
+    while True:
+        # Hash each file in the sample paths (if it exists).
+        source_sample_paths_subset = tuple(
+            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT)
+        )
+        if not source_sample_paths_subset:
+            break
+
+        hash_output = borgmatic.execute.execute_command_and_capture_output(
+            (spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
+            + tuple(
+                path for path in source_sample_paths_subset if path in existing_source_sample_paths
+            ),
+            working_directory=working_directory,
+        )
+
+        source_hashes.update(
+            **dict(
+                (reversed(line.split('  ', 1)) for line in hash_output.splitlines()),
+                # Represent non-existent files as having empty hashes so the comparison below still works.
+                **{
+                    path: ''
+                    for path in source_sample_paths_subset
+                    if path not in existing_source_sample_paths
+                },
+            )
+        )
+
+        # Get the hash for each file in the archive.
+        archive_hashes.update(
+            **dict(
+                reversed(line.split(' ', 1))
+                for line in borgmatic.borg.list.capture_archive_listing(
+                    repository['path'],
+                    archive,
+                    config,
+                    local_borg_version,
+                    global_arguments,
+                    list_paths=source_sample_paths_subset,
+                    path_format='{xxh64} {path}{NUL}',  # noqa: FS003
+                    local_path=local_path,
+                    remote_path=remote_path,
+                )
+                if line
+            )
+        )
+
+    # Compare the source hashes with the archive hashes to see how many match.
+    failing_paths = []
+
+    for path, source_hash in source_hashes.items():
+        archive_hash = archive_hashes.get(path.lstrip(os.path.sep))
+
+        if archive_hash is not None and archive_hash == source_hash:
+            continue
+
+        failing_paths.append(path)
+
+    return tuple(failing_paths)
+
+
+def spot_check(
+    repository,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
+    borgmatic_runtime_directory,
+):
+    '''
+    Given a repository dict, a loaded configuration dict, the local Borg version, global arguments
+    as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the borgmatic
+    runtime directory, perform a spot check for the latest archive in the given repository.
+
+    A spot check compares file counts and also the hashes for a random sampling of source files on
+    disk to those stored in the latest archive. If any differences are beyond configured tolerances,
+    then the check fails.
+    '''
+    logger.debug('Running spot check')
+
+    try:
+        spot_check_config = next(
+            check for check in config.get('checks', ()) if check.get('name') == 'spot'
+        )
+    except StopIteration:
+        raise ValueError('Cannot run spot check because it is unconfigured')
+
+    if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
+        raise ValueError(
+            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage'
+        )
+
+    source_paths = collect_spot_check_source_paths(
+        repository,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        borgmatic_runtime_directory,
+    )
+    logger.debug(f'{len(source_paths)} total source paths for spot check')
+
+    archive = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        'latest',
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    logger.debug(f'Using archive {archive} for spot check')
+
+    archive_paths = collect_spot_check_archive_paths(
+        repository,
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        borgmatic_runtime_directory,
+    )
+    logger.debug(f'{len(archive_paths)} total archive paths for spot check')
+
+    if len(source_paths) == 0:
+        logger.debug(
+            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}'
+        )
+        raise ValueError(
+            'Spot check failed: There are no source paths to compare against the archive'
+        )
+
+    # Calculate the percentage delta between the source paths count and the archive paths count, and
+    # compare that delta to the configured count tolerance percentage.
+    count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
+
+    if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
+        rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
+        logger.debug(
+            f'Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}'
+        )
+        logger.debug(
+            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}'
+        )
+        raise ValueError(
+            f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
+        )
+
+    failing_paths = compare_spot_check_hashes(
+        repository,
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        source_paths,
+    )
+
+    # Error if the percentage of failing hashes exceeds the configured tolerance percentage.
+    logger.debug(f'{len(failing_paths)} non-matching spot check hashes')
+    data_tolerance_percentage = spot_check_config['data_tolerance_percentage']
+    failing_percentage = (len(failing_paths) / len(source_paths)) * 100
+
+    if failing_percentage > data_tolerance_percentage:
+        logger.debug(
+            f'Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
+        )
+        raise ValueError(
+            f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
+        )
+
+    logger.info(
+        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
+    )
+
+
 def run_check(
     config_filename,
     repository,
-    location,
-    storage,
-    consistency,
-    hooks,
+    config,
     hook_context,
     local_borg_version,
     check_arguments,
@@ -23,6 +691,8 @@ def run_check(
 ):
     '''
     Run the "check" action for the given repository.
+
+    Raise ValueError if the Borg repository ID cannot be determined.
     '''
     if check_arguments.repository and not borgmatic.config.validate.repositories_match(
         repository, check_arguments.repository
@@ -30,31 +700,82 @@ def run_check(
         return
 
     borgmatic.hooks.command.execute_hook(
-        hooks.get('before_check'),
-        hooks.get('umask'),
+        config.get('before_check'),
+        config.get('umask'),
         config_filename,
         'pre-check',
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
-    borgmatic.borg.check.check_archives(
+
+    logger.info('Running consistency checks')
+
+    repository_id = borgmatic.borg.check.get_repository_id(
         repository['path'],
-        location,
-        storage,
-        consistency,
+        config,
         local_borg_version,
         global_arguments,
         local_path=local_path,
         remote_path=remote_path,
-        progress=check_arguments.progress,
-        repair=check_arguments.repair,
-        only_checks=check_arguments.only,
-        force=check_arguments.force,
     )
+    upgrade_check_times(config, repository_id)
+    configured_checks = parse_checks(config, check_arguments.only_checks)
+    archive_filter_flags = borgmatic.borg.check.make_archive_filter_flags(
+        local_borg_version, config, configured_checks, check_arguments
+    )
+    archives_check_id = make_archives_check_id(archive_filter_flags)
+    checks = filter_checks_on_frequency(
+        config,
+        repository_id,
+        configured_checks,
+        check_arguments.force,
+        archives_check_id,
+    )
+    borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
+
+    if borg_specific_checks:
+        borgmatic.borg.check.check_archives(
+            repository['path'],
+            config,
+            local_borg_version,
+            check_arguments,
+            global_arguments,
+            borg_specific_checks,
+            archive_filter_flags,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+        for check in borg_specific_checks:
+            write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
+
+    if 'extract' in checks:
+        borgmatic.borg.extract.extract_last_archive_dry_run(
+            config,
+            local_borg_version,
+            global_arguments,
+            repository['path'],
+            config.get('lock_wait'),
+            local_path,
+            remote_path,
+        )
+        write_check_time(make_check_time_path(config, repository_id, 'extract'))
+
+    if 'spot' in checks:
+        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+            spot_check(
+                repository,
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path,
+                remote_path,
+                borgmatic_runtime_directory,
+            )
+        write_check_time(make_check_time_path(config, repository_id, 'spot'))
+
     borgmatic.hooks.command.execute_hook(
-        hooks.get('after_check'),
-        hooks.get('umask'),
+        config.get('after_check'),
+        config.get('umask'),
         config_filename,
         'post-check',
         global_arguments.dry_run,

borgmatic/actions/compact.py

@@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
 def run_compact(
     config_filename,
     repository,
-    storage,
-    retention,
-    hooks,
+    config,
     hook_context,
     local_borg_version,
     compact_arguments,
@@ -31,21 +29,19 @@ def run_compact(
         return
 
     borgmatic.hooks.command.execute_hook(
-        hooks.get('before_compact'),
-        hooks.get('umask'),
+        config.get('before_compact'),
+        config.get('umask'),
         config_filename,
         'pre-compact',
         global_arguments.dry_run,
         **hook_context,
     )
     if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
-        logger.info(
-            f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}'
-        )
+        logger.info(f'Compacting segments{dry_run_label}')
         borgmatic.borg.compact.compact_segments(
             global_arguments.dry_run,
             repository['path'],
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path=local_path,
@@ -55,12 +51,11 @@ def run_compact(
             threshold=compact_arguments.threshold,
         )
     else:  # pragma: nocover
-        logger.info(
-            f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)'
-        )
+        logger.info('Skipping compact (only available/needed in Borg 1.2+)')
+
     borgmatic.hooks.command.execute_hook(
-        hooks.get('after_compact'),
-        hooks.get('umask'),
+        config.get('after_compact'),
+        config.get('umask'),
         config_filename,
         'post-compact',
         global_arguments.dry_run,

borgmatic/actions/config/bootstrap.py

@@ -0,0 +1,131 @@
+import json
+import logging
+import os
+
+import borgmatic.borg.extract
+import borgmatic.borg.repo_list
+import borgmatic.config.paths
+import borgmatic.config.validate
+import borgmatic.hooks.command
+
+logger = logging.getLogger(__name__)
+
+
+def make_bootstrap_config(bootstrap_arguments):
+    '''
+    Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
+    '''
+    return {
+        'ssh_command': bootstrap_arguments.ssh_command,
+        # In case the repo has been moved or is accessed from a different path at the point of
+        # bootstrapping.
+        'relocated_repo_access_is_ok': True,
+    }
+
+
+def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
+    repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
+    borgmatic source directory, destination directory, and whether to strip components), the global
+    arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
+    return the config paths from the manifest.json file in the borgmatic source directory or runtime
+    directory after extracting it from the repository archive.
+
+    Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
+    expected configuration path data.
+    '''
+    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
+        {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
+    )
+    config = make_bootstrap_config(bootstrap_arguments)
+
+    # Probe for the manifest file in multiple locations, as the default location has moved to the
+    # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
+    # still want to support reading the manifest from previously created archives as well.
+    with borgmatic.config.paths.Runtime_directory(
+        {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
+    ) as borgmatic_runtime_directory:
+        for base_directory in (
+            'borgmatic',
+            borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+            borgmatic_source_directory,
+        ):
+            borgmatic_manifest_path = 'sh:' + os.path.join(
+                base_directory, 'bootstrap', 'manifest.json'
+            )
+
+            extract_process = borgmatic.borg.extract.extract_archive(
+                global_arguments.dry_run,
+                bootstrap_arguments.repository,
+                archive_name,
+                [borgmatic_manifest_path],
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path=bootstrap_arguments.local_path,
+                remote_path=bootstrap_arguments.remote_path,
+                extract_to_stdout=True,
+            )
+            manifest_json = extract_process.stdout.read()
+
+            if manifest_json:
+                break
+        else:
+            raise ValueError(
+                'Cannot read configuration paths from archive due to missing bootstrap manifest'
+            )
+
+    try:
+        manifest_data = json.loads(manifest_json)
+    except json.JSONDecodeError as error:
+        raise ValueError(
+            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
+        )
+
+    try:
+        return manifest_data['config_paths']
+    except KeyError:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to invalid bootstrap manifest'
+        )
+
+
+def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Run the "bootstrap" action for the given repository.
+
+    Raise ValueError if the bootstrap configuration could not be loaded.
+    Raise CalledProcessError or OSError if Borg could not be run.
+    '''
+    config = make_bootstrap_config(bootstrap_arguments)
+    archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+        bootstrap_arguments.repository,
+        bootstrap_arguments.archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=bootstrap_arguments.local_path,
+        remote_path=bootstrap_arguments.remote_path,
+    )
+    manifest_config_paths = get_config_paths(
+        archive_name, bootstrap_arguments, global_arguments, local_borg_version
+    )
+
+    logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
+
+    borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        bootstrap_arguments.repository,
+        archive_name,
+        [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=bootstrap_arguments.local_path,
+        remote_path=bootstrap_arguments.remote_path,
+        extract_to_stdout=False,
+        destination_path=bootstrap_arguments.destination,
+        strip_components=bootstrap_arguments.strip_components,
+        progress=bootstrap_arguments.progress,
+    )

borgmatic/actions/config/generate.py

@@ -0,0 +1,48 @@
+import logging
+
+import borgmatic.config.generate
+import borgmatic.config.validate
+import borgmatic.logger
+
+logger = logging.getLogger(__name__)
+
+
+def run_generate(generate_arguments, global_arguments):
+    '''
+    Given the generate arguments and the global arguments, each as an argparse.Namespace instance,
+    run the "generate" action.
+
+    Raise FileExistsError if a file already exists at the destination path and the generate
+    arguments do not have overwrite set.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+    dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
+
+    logger.answer(
+        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
+    )
+
+    borgmatic.config.generate.generate_sample_configuration(
+        global_arguments.dry_run,
+        generate_arguments.source_filename,
+        generate_arguments.destination_filename,
+        borgmatic.config.validate.schema_filename(),
+        overwrite=generate_arguments.overwrite,
+    )
+
+    if generate_arguments.source_filename:
+        logger.answer(
+            f'''
+Merged in the contents of configuration file at: {generate_arguments.source_filename}
+To review the changes made, run:
+
+    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
+        )
+
+    logger.answer(
+        '''
+This includes all available configuration options with example values, the few
+required options as indicated. Please edit the file to suit your needs.
+
+If you ever need help: https://torsion.org/borgmatic/#issues'''
+    )

borgmatic/actions/config/validate.py

@@ -0,0 +1,25 @@
+import logging
+
+import borgmatic.config.generate
+import borgmatic.logger
+
+logger = logging.getLogger(__name__)
+
+
+def run_validate(validate_arguments, configs):
+    '''
+    Given the validate arguments as an argparse.Namespace instance and a dict of configuration
+    filename to corresponding parsed configuration, run the "validate" action.
+
+    Most of the validation is actually performed implicitly by the standard borgmatic configuration
+    loading machinery prior to here, so this function mainly exists to support additional validate
+    flags like "--show".
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    if validate_arguments.show:
+        for config_path, config in configs.items():
+            if len(configs) > 1:
+                logger.answer('---')
+
+            logger.answer(borgmatic.config.generate.render_configuration(config))

borgmatic/actions/create.py

@@ -1,21 +1,266 @@
-import json
+import glob
+import itertools
 import logging
+import os
+import pathlib
 
+import borgmatic.actions.json
 import borgmatic.borg.create
+import borgmatic.borg.pattern
+import borgmatic.config.paths
 import borgmatic.config.validate
 import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
-import borgmatic.hooks.dump
 
 logger = logging.getLogger(__name__)
 
 
+def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
+    '''
+    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
+    return it.
+    '''
+    try:
+        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
+    except ValueError:
+        raise ValueError(f'Invalid pattern: {pattern_line}')
+
+    try:
+        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
+        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
+    except ValueError:
+        pattern_style = default_style
+        path = remainder
+
+    return borgmatic.borg.pattern.Pattern(
+        path,
+        borgmatic.borg.pattern.Pattern_type(pattern_type),
+        borgmatic.borg.pattern.Pattern_style(pattern_style),
+        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
+    )
+
+
+def collect_patterns(config):
+    '''
+    Given a configuration dict, produce a single sequence of patterns comprised of the configured
+    source directories, patterns, excludes, pattern files, and exclude files.
+
+    The idea is that Borg has all these different ways of specifying includes, excludes, source
+    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
+    ease of manipulation within borgmatic.
+    '''
+    try:
+        return (
+            tuple(
+                borgmatic.borg.pattern.Pattern(
+                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
+                )
+                for source_directory in config.get('source_directories', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip())
+                for pattern_line in config.get('patterns', ())
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for exclude_line in config.get('exclude_patterns', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip())
+                for filename in config.get('patterns_from', ())
+                for pattern_line in open(filename).readlines()
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for filename in config.get('exclude_from', ())
+                for exclude_line in open(filename).readlines()
+                if not exclude_line.lstrip().startswith('#')
+                if exclude_line.strip()
+            )
+        )
+    except (FileNotFoundError, OSError) as error:
+        logger.debug(error)
+
+        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
+
+
+def expand_directory(directory, working_directory):
+    '''
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+
+    Take into account the given working directory so that relative paths are supported.
+    '''
+    expanded_directory = os.path.expanduser(directory)
+
+    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
+    # only available in Python 3.10+.
+    normalized_directory = os.path.join(working_directory or '', expanded_directory)
+    glob_paths = glob.glob(normalized_directory)
+
+    if not glob_paths:
+        return [expanded_directory]
+
+    working_directory_prefix = os.path.join(working_directory or '', '')
+
+    return [
+        (
+            glob_path
+            # If these are equal, that means we didn't add any working directory prefix above.
+            if normalized_directory == expanded_directory
+            # Remove the working directory prefix that we added above in order to make glob() work.
+            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
+            # hack.
+            else glob_path.removeprefix(working_directory_prefix)
+        )
+        for glob_path in glob_paths
+    ]
+
+
+def expand_patterns(patterns, working_directory=None, skip_paths=None):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    expand tildes and globs in each root pattern. Return all the resulting patterns (not just the
+    root patterns) as a tuple.
+
+    If a set of paths are given to skip, then don't expand any patterns matching them.
+    '''
+    if patterns is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(
+            (
+                (
+                    borgmatic.borg.pattern.Pattern(
+                        expanded_path,
+                        pattern.type,
+                        pattern.style,
+                        pattern.device,
+                        pattern.source,
+                    )
+                    for expanded_path in expand_directory(pattern.path, working_directory)
+                )
+                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.path not in (skip_paths or ())
+                else (pattern,)
+            )
+            for pattern in patterns
+        )
+    )
+
+
+def device_map_patterns(patterns, working_directory=None):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    determine the identifier for the device on which the pattern's path resides—or None if the path
+    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
+    device field populated. But if the device field is already set, don't bother setting it again.
+
+    This is handy for determining whether two different pattern paths are on the same filesystem
+    (have the same device identifier).
+    '''
+    return tuple(
+        borgmatic.borg.pattern.Pattern(
+            pattern.path,
+            pattern.type,
+            pattern.style,
+            device=pattern.device
+            or (
+                os.stat(full_path).st_dev
+                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and os.path.exists(full_path)
+                else None
+            ),
+            source=pattern.source,
+        )
+        for pattern in patterns
+        for full_path in (os.path.join(working_directory or '', pattern.path),)
+    )
+
+
+def deduplicate_patterns(patterns):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
+    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
+    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
+    modification.
+
+    The one exception to deduplication is two paths are on different filesystems (devices). In that
+    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
+    one_file_system option is true).
+
+    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
+    child patterns, because it will naturally spider the contents of the parent pattern's path. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = {}  # Use just the keys as an ordered set.
+
+    for pattern in patterns:
+        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+            deduplicated[pattern] = True
+            continue
+
+        parents = pathlib.PurePath(pattern.path).parents
+
+        # If another directory in the given list is a parent of current directory (even n levels up)
+        # and both are on the same filesystem, then the current directory is a duplicate.
+        for other_pattern in patterns:
+            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+                continue
+
+            if any(
+                pathlib.PurePath(other_pattern.path) == parent
+                and pattern.device is not None
+                and other_pattern.device == pattern.device
+                for parent in parents
+            ):
+                break
+        else:
+            deduplicated[pattern] = True
+
+    return tuple(deduplicated.keys())
+
+
+def process_patterns(patterns, working_directory, skip_expand_paths=None):
+    '''
+    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
+    "root" patterns, returning the resulting root and non-root patterns as a list.
+
+    If any paths are given to skip, don't expand them.
+    '''
+    skip_paths = set(skip_expand_paths or ())
+
+    return list(
+        deduplicate_patterns(
+            device_map_patterns(
+                expand_patterns(
+                    patterns,
+                    working_directory=working_directory,
+                    skip_paths=skip_paths,
+                )
+            )
+        )
+    )
+
+
 def run_create(
     config_filename,
     repository,
-    location,
-    storage,
-    hooks,
+    config,
+    config_paths,
     hook_context,
     local_borg_version,
     create_arguments,
@@ -35,61 +280,73 @@ def run_create(
         return
 
     borgmatic.hooks.command.execute_hook(
-        hooks.get('before_backup'),
-        hooks.get('umask'),
+        config.get('before_backup'),
+        config.get('umask'),
         config_filename,
         'pre-backup',
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
-    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-        'remove_database_dumps',
-        hooks,
-        repository['path'],
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        location,
-        global_arguments.dry_run,
-    )
-    active_dumps = borgmatic.hooks.dispatch.call_hooks(
-        'dump_databases',
-        hooks,
-        repository['path'],
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        location,
-        global_arguments.dry_run,
-    )
-    stream_processes = [process for processes in active_dumps.values() for process in processes]
 
-    json_output = borgmatic.borg.create.create_archive(
-        global_arguments.dry_run,
-        repository['path'],
-        location,
-        storage,
-        local_borg_version,
-        global_arguments,
-        local_path=local_path,
-        remote_path=remote_path,
-        progress=create_arguments.progress,
-        stats=create_arguments.stats,
-        json=create_arguments.json,
-        list_files=create_arguments.list_files,
-        stream_processes=stream_processes,
-    )
-    if json_output:  # pragma: nocover
-        yield json.loads(json_output)
+    logger.info(f'Creating archive{dry_run_label}')
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            'remove_data_source_dumps',
+            config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            borgmatic_runtime_directory,
+            global_arguments.dry_run,
+        )
+        patterns = process_patterns(collect_patterns(config), working_directory)
+        active_dumps = borgmatic.hooks.dispatch.call_hooks(
+            'dump_data_sources',
+            config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            config_paths,
+            borgmatic_runtime_directory,
+            patterns,
+            global_arguments.dry_run,
+        )
+
+        # Process the patterns again in case any data source hooks updated them. Without this step,
+        # we could end up with duplicate paths that cause Borg to hang when it tries to read from
+        # the same named pipe twice.
+        patterns = process_patterns(patterns, working_directory, skip_expand_paths=config_paths)
+        stream_processes = [process for processes in active_dumps.values() for process in processes]
+
+        json_output = borgmatic.borg.create.create_archive(
+            global_arguments.dry_run,
+            repository['path'],
+            config,
+            patterns,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+            local_path=local_path,
+            remote_path=remote_path,
+            progress=create_arguments.progress,
+            stats=create_arguments.stats,
+            json=create_arguments.json,
+            list_files=create_arguments.list_files,
+            stream_processes=stream_processes,
+        )
+
+        if json_output:
+            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+
+        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            'remove_data_source_dumps',
+            config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            borgmatic_runtime_directory,
+            global_arguments.dry_run,
+        )
 
-    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-        'remove_database_dumps',
-        hooks,
-        config_filename,
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        location,
-        global_arguments.dry_run,
-    )
     borgmatic.hooks.command.execute_hook(
-        hooks.get('after_backup'),
-        hooks.get('umask'),
+        config.get('after_backup'),
+        config.get('umask'),
         config_filename,
         'post-backup',
         global_arguments.dry_run,

borgmatic/actions/delete.py

@@ -0,0 +1,50 @@
+import logging
+
+import borgmatic.actions.arguments
+import borgmatic.borg.delete
+import borgmatic.borg.repo_delete
+import borgmatic.borg.repo_list
+
+logger = logging.getLogger(__name__)
+
+
+def run_delete(
+    repository,
+    config,
+    local_borg_version,
+    delete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "delete" action for the given repository and archive(s).
+    '''
+    if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, delete_arguments.repository
+    ):
+        logger.answer('Deleting archives')
+
+        archive_name = (
+            borgmatic.borg.repo_list.resolve_archive_name(
+                repository['path'],
+                delete_arguments.archive,
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+            if delete_arguments.archive
+            else None
+        )
+
+        borgmatic.borg.delete.delete_archives(
+            repository,
+            config,
+            local_borg_version,
+            borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
+            global_arguments,
+            local_path,
+            remote_path,
+        )

borgmatic/actions/export_key.py

@@ -0,0 +1,33 @@
+import logging
+
+import borgmatic.borg.export_key
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_export_key(
+    repository,
+    config,
+    local_borg_version,
+    export_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "key export" action for the given repository.
+    '''
+    if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, export_arguments.repository
+    ):
+        logger.info('Exporting repository key')
+        borgmatic.borg.export_key.export_key(
+            repository['path'],
+            config,
+            local_borg_version,
+            export_arguments,
+            global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+        )

borgmatic/actions/export_tar.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.export_tar
-import borgmatic.borg.rlist
+import borgmatic.borg.repo_list
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 def run_export_tar(
     repository,
-    storage,
+    config,
     local_borg_version,
     export_tar_arguments,
     global_arguments,
@@ -22,16 +22,14 @@ def run_export_tar(
     if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, export_tar_arguments.repository
     ):
-        logger.info(
-            f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file'
-        )
+        logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
         borgmatic.borg.export_tar.export_tar_archive(
             global_arguments.dry_run,
             repository['path'],
-            borgmatic.borg.rlist.resolve_archive_name(
+            borgmatic.borg.repo_list.resolve_archive_name(
                 repository['path'],
                 export_tar_arguments.archive,
-                storage,
+                config,
                 local_borg_version,
                 global_arguments,
                 local_path,
@@ -39,7 +37,7 @@ def run_export_tar(
             ),
             export_tar_arguments.paths,
             export_tar_arguments.destination,
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path=local_path,

borgmatic/actions/extract.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.extract
-import borgmatic.borg.rlist
+import borgmatic.borg.repo_list
 import borgmatic.config.validate
 import borgmatic.hooks.command
 
@@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
 def run_extract(
     config_filename,
     repository,
-    location,
-    storage,
-    hooks,
+    config,
     hook_context,
     local_borg_version,
     extract_arguments,
@@ -25,8 +23,8 @@ def run_extract(
     Run the "extract" action for the given repository.
     '''
     borgmatic.hooks.command.execute_hook(
-        hooks.get('before_extract'),
-        hooks.get('umask'),
+        config.get('before_extract'),
+        config.get('umask'),
         config_filename,
         'pre-extract',
         global_arguments.dry_run,
@@ -35,24 +33,21 @@ def run_extract(
     if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, extract_arguments.repository
     ):
-        logger.info(
-            f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}'
-        )
+        logger.info(f'Extracting archive {extract_arguments.archive}')
         borgmatic.borg.extract.extract_archive(
             global_arguments.dry_run,
             repository['path'],
-            borgmatic.borg.rlist.resolve_archive_name(
+            borgmatic.borg.repo_list.resolve_archive_name(
                 repository['path'],
                 extract_arguments.archive,
-                storage,
+                config,
                 local_borg_version,
                 global_arguments,
                 local_path,
                 remote_path,
             ),
             extract_arguments.paths,
-            location,
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path=local_path,
@@ -62,8 +57,8 @@ def run_extract(
             progress=extract_arguments.progress,
         )
     borgmatic.hooks.command.execute_hook(
-        hooks.get('after_extract'),
-        hooks.get('umask'),
+        config.get('after_extract'),
+        config.get('umask'),
         config_filename,
         'post-extract',
         global_arguments.dry_run,

borgmatic/actions/info.py

@@ -1,9 +1,9 @@
-import json
 import logging
 
 import borgmatic.actions.arguments
+import borgmatic.actions.json
 import borgmatic.borg.info
-import borgmatic.borg.rlist
+import borgmatic.borg.repo_list
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 def run_info(
     repository,
-    storage,
+    config,
     local_borg_version,
     info_arguments,
     global_arguments,
@@ -26,14 +26,12 @@ def run_info(
     if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, info_arguments.repository
     ):
-        if not info_arguments.json:  # pragma: nocover
-            logger.answer(
-                f'{repository.get("label", repository["path"])}: Displaying archive summary information'
-            )
-        archive_name = borgmatic.borg.rlist.resolve_archive_name(
+        if not info_arguments.json:
+            logger.answer('Displaying archive summary information')
+        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
             info_arguments.archive,
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path,
@@ -41,12 +39,12 @@ def run_info(
         )
         json_output = borgmatic.borg.info.display_archives_info(
             repository['path'],
-            storage,
+            config,
             local_borg_version,
             borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
             global_arguments,
             local_path,
             remote_path,
         )
-        if json_output:  # pragma: nocover
-            yield json.loads(json_output)
+        if json_output:
+            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/json.py

@@ -0,0 +1,30 @@
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def parse_json(borg_json_output, label):
+    '''
+    Given a Borg JSON output string, parse it as JSON into a dict. Inject the given borgmatic
+    repository label into it and return the dict.
+
+    Raise JSONDecodeError if the JSON output cannot be parsed.
+    '''
+    lines = borg_json_output.splitlines()
+    start_line_index = 0
+
+    # Scan forward to find the first line starting with "{" and assume that's where the JSON starts.
+    for line_index, line in enumerate(lines):
+        if line.startswith('{'):
+            start_line_index = line_index
+            break
+
+    json_data = json.loads('\n'.join(lines[start_line_index:]))
+
+    if 'repository' not in json_data:
+        return json_data
+
+    json_data['repository']['label'] = label or ''
+
+    return json_data

borgmatic/actions/list.py

@@ -1,7 +1,7 @@
-import json
 import logging
 
 import borgmatic.actions.arguments
+import borgmatic.actions.json
 import borgmatic.borg.list
 import borgmatic.config.validate
 
@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
 
 def run_list(
     repository,
-    storage,
+    config,
     local_borg_version,
     list_arguments,
     global_arguments,
@@ -25,16 +25,16 @@ def run_list(
     if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, list_arguments.repository
     ):
-        if not list_arguments.json:  # pragma: nocover
-            if list_arguments.find_paths:
-                logger.answer(f'{repository.get("label", repository["path"])}: Searching archives')
-            elif not list_arguments.archive:
-                logger.answer(f'{repository.get("label", repository["path"])}: Listing archives')
+        if not list_arguments.json:
+            if list_arguments.find_paths:  # pragma: no cover
+                logger.answer('Searching archives')
+            elif not list_arguments.archive:  # pragma: no cover
+                logger.answer('Listing archives')
 
-        archive_name = borgmatic.borg.rlist.resolve_archive_name(
+        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
             list_arguments.archive,
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path,
@@ -42,12 +42,12 @@ def run_list(
         )
         json_output = borgmatic.borg.list.list_archive(
             repository['path'],
-            storage,
+            config,
             local_borg_version,
             borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
             global_arguments,
             local_path,
             remote_path,
         )
-        if json_output:  # pragma: nocover
-            yield json.loads(json_output)
+        if json_output:
+            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/mount.py

@@ -1,7 +1,7 @@
 import logging
 
 import borgmatic.borg.mount
-import borgmatic.borg.rlist
+import borgmatic.borg.repo_list
 import borgmatic.config.validate
 
 logger = logging.getLogger(__name__)
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 def run_mount(
     repository,
-    storage,
+    config,
     local_borg_version,
     mount_arguments,
     global_arguments,
@@ -23,25 +23,23 @@ def run_mount(
         repository, mount_arguments.repository
     ):
         if mount_arguments.archive:
-            logger.info(
-                f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}'
-            )
+            logger.info(f'Mounting archive {mount_arguments.archive}')
         else:  # pragma: nocover
-            logger.info(f'{repository.get("label", repository["path"])}: Mounting repository')
+            logger.info('Mounting repository')
 
         borgmatic.borg.mount.mount_archive(
             repository['path'],
-            borgmatic.borg.rlist.resolve_archive_name(
+            borgmatic.borg.repo_list.resolve_archive_name(
                 repository['path'],
                 mount_arguments.archive,
-                storage,
+                config,
                 local_borg_version,
                 global_arguments,
                 local_path,
                 remote_path,
             ),
             mount_arguments,
-            storage,
+            config,
             local_borg_version,
             global_arguments,
             local_path=local_path,

borgmatic/actions/prune.py

@@ -10,9 +10,7 @@ logger = logging.getLogger(__name__)
 def run_prune(
     config_filename,
     repository,
-    storage,
-    retention,
-    hooks,
+    config,
     hook_context,
     local_borg_version,
     prune_arguments,
@@ -30,19 +28,18 @@ def run_prune(
         return
 
     borgmatic.hooks.command.execute_hook(
-        hooks.get('before_prune'),
-        hooks.get('umask'),
+        config.get('before_prune'),
+        config.get('umask'),
         config_filename,
         'pre-prune',
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}')
+    logger.info(f'Pruning archives{dry_run_label}')
     borgmatic.borg.prune.prune_archives(
         global_arguments.dry_run,
         repository['path'],
-        storage,
-        retention,
+        config,
         local_borg_version,
         prune_arguments,
         global_arguments,
@@ -50,8 +47,8 @@ def run_prune(
         remote_path=remote_path,
     )
     borgmatic.hooks.command.execute_hook(
-        hooks.get('after_prune'),
-        hooks.get('umask'),
+        config.get('after_prune'),
+        config.get('umask'),
         config_filename,
         'post-prune',
         global_arguments.dry_run,

borgmatic/actions/rcreate.py

@@ -1,41 +0,0 @@
-import logging
-
-import borgmatic.borg.rcreate
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_rcreate(
-    repository,
-    storage,
-    local_borg_version,
-    rcreate_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "rcreate" action for the given repository.
-    '''
-    if rcreate_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, rcreate_arguments.repository
-    ):
-        return
-
-    logger.info(f'{repository.get("label", repository["path"])}: Creating repository')
-    borgmatic.borg.rcreate.create_repository(
-        global_arguments.dry_run,
-        repository['path'],
-        storage,
-        local_borg_version,
-        global_arguments,
-        rcreate_arguments.encryption_mode,
-        rcreate_arguments.source_repository,
-        rcreate_arguments.copy_crypt_key,
-        rcreate_arguments.append_only,
-        rcreate_arguments.storage_quota,
-        rcreate_arguments.make_parent_dirs,
-        local_path=local_path,
-        remote_path=remote_path,
-    )

borgmatic/actions/repo_create.py

@@ -0,0 +1,41 @@
+import logging
+
+import borgmatic.borg.repo_create
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_repo_create(
+    repository,
+    config,
+    local_borg_version,
+    repo_create_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "repo-create" action for the given repository.
+    '''
+    if repo_create_arguments.repository and not borgmatic.config.validate.repositories_match(
+        repository, repo_create_arguments.repository
+    ):
+        return
+
+    logger.info('Creating repository')
+    borgmatic.borg.repo_create.create_repository(
+        global_arguments.dry_run,
+        repository['path'],
+        config,
+        local_borg_version,
+        global_arguments,
+        repo_create_arguments.encryption_mode,
+        repo_create_arguments.source_repository,
+        repo_create_arguments.copy_crypt_key,
+        repo_create_arguments.append_only,
+        repo_create_arguments.storage_quota,
+        repo_create_arguments.make_parent_dirs,
+        local_path=local_path,
+        remote_path=remote_path,
+    )

borgmatic/actions/repo_delete.py

@@ -0,0 +1,35 @@
+import logging
+
+import borgmatic.borg.repo_delete
+
+logger = logging.getLogger(__name__)
+
+
+def run_repo_delete(
+    repository,
+    config,
+    local_borg_version,
+    repo_delete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "repo-delete" action for the given repository.
+    '''
+    if repo_delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, repo_delete_arguments.repository
+    ):
+        logger.answer(
+            'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '')
+        )
+
+        borgmatic.borg.repo_delete.delete_repository(
+            repository,
+            config,
+            local_borg_version,
+            repo_delete_arguments,
+            global_arguments,
+            local_path,
+            remote_path,
+        )

borgmatic/actions/repo_info.py

@@ -0,0 +1,40 @@
+import logging
+
+import borgmatic.actions.json
+import borgmatic.borg.repo_info
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_repo_info(
+    repository,
+    config,
+    local_borg_version,
+    repo_info_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "repo-info" action for the given repository.
+
+    If repo_info_arguments.json is True, yield the JSON output from the info for the repository.
+    '''
+    if repo_info_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, repo_info_arguments.repository
+    ):
+        if not repo_info_arguments.json:
+            logger.answer('Displaying repository summary information')
+
+        json_output = borgmatic.borg.repo_info.display_repository_info(
+            repository['path'],
+            config,
+            local_borg_version,
+            repo_info_arguments=repo_info_arguments,
+            global_arguments=global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+        if json_output:
+            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/repo_list.py

@@ -0,0 +1,40 @@
+import logging
+
+import borgmatic.actions.json
+import borgmatic.borg.repo_list
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_repo_list(
+    repository,
+    config,
+    local_borg_version,
+    repo_list_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "repo-list" action for the given repository.
+
+    If repo_list_arguments.json is True, yield the JSON output from listing the repository.
+    '''
+    if repo_list_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, repo_list_arguments.repository
+    ):
+        if not repo_list_arguments.json:
+            logger.answer('Listing repository')
+
+        json_output = borgmatic.borg.repo_list.list_repository(
+            repository['path'],
+            config,
+            local_borg_version,
+            repo_list_arguments=repo_list_arguments,
+            global_arguments=global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+        if json_output:
+            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))

borgmatic/actions/restore.py

@@ -1,254 +1,429 @@
-import copy
+import collections
 import logging
 import os
+import pathlib
+import shutil
+import tempfile
 
 import borgmatic.borg.extract
 import borgmatic.borg.list
 import borgmatic.borg.mount
-import borgmatic.borg.rlist
-import borgmatic.borg.state
+import borgmatic.borg.repo_list
+import borgmatic.config.paths
 import borgmatic.config.validate
+import borgmatic.hooks.data_source.dump
 import borgmatic.hooks.dispatch
-import borgmatic.hooks.dump
 
 logger = logging.getLogger(__name__)
 
 
-UNSPECIFIED_HOOK = object()
+UNSPECIFIED = object()
 
 
-def get_configured_database(
-    hooks, archive_database_names, hook_name, database_name, configuration_database_name=None
-):
+Dump = collections.namedtuple(
+    'Dump',
+    ('hook_name', 'data_source_name', 'hostname', 'port'),
+    defaults=('localhost', None),
+)
+
+
+def dumps_match(first, second, default_port=None):
     '''
-    Find the first database with the given hook name and database name in the configured hooks
-    dict and the given archive database names dict (from hook name to database names contained in
-    a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all database
-    hooks for the named database. If a configuration database name is given, use that instead of the
-    database name to lookup the database in the given hooks configuration.
-
-    Return the found database as a tuple of (found hook name, database configuration dict).
+    Compare two Dump instances for equality while supporting a field value of UNSPECIFIED, which
+    indicates that the field should match any value. If a default port is given, then consider any
+    dump having that port to match with a dump having a None port.
     '''
-    if not configuration_database_name:
-        configuration_database_name = database_name
+    for field_name in first._fields:
+        first_value = getattr(first, field_name)
+        second_value = getattr(second, field_name)
 
-    if hook_name == UNSPECIFIED_HOOK:
-        hooks_to_search = hooks
+        if default_port is not None and field_name == 'port':
+            if first_value == default_port and second_value is None:
+                continue
+            if second_value == default_port and first_value is None:
+                continue
+
+        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
+            continue
+
+        if first_value != second_value:
+            return False
+
+    return True
+
+
+def render_dump_metadata(dump):
+    '''
+    Given a Dump instance, make a display string describing it for use in log messages.
+    '''
+    name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
+    hostname = dump.hostname or UNSPECIFIED
+    port = None if dump.port is UNSPECIFIED else dump.port
+
+    if port:
+        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
     else:
-        hooks_to_search = {hook_name: hooks[hook_name]}
+        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
 
-    return next(
-        (
-            (name, hook_database)
-            for (name, hook) in hooks_to_search.items()
-            for hook_database in hook
-            if hook_database['name'] == configuration_database_name
-            and database_name in archive_database_names.get(name, [])
-        ),
-        (None, None),
+    if dump.hook_name not in (None, UNSPECIFIED):
+        return f'{metadata} ({dump.hook_name})'
+
+    return metadata
+
+
+def get_configured_data_source(config, restore_dump):
+    '''
+    Search in the given configuration dict for dumps corresponding to the given dump to restore. If
+    there are multiple matches, error.
+
+    Return the found data source as a data source configuration dict or None if not found.
+    '''
+    try:
+        hooks_to_search = {restore_dump.hook_name: config[restore_dump.hook_name]}
+    except KeyError:
+        return None
+
+    matching_dumps = tuple(
+        hook_data_source
+        for (hook_name, hook_config) in hooks_to_search.items()
+        for hook_data_source in hook_config
+        for default_port in (
+            borgmatic.hooks.dispatch.call_hook(
+                function_name='get_default_port',
+                config=config,
+                hook_name=hook_name,
+            ),
+        )
+        if dumps_match(
+            Dump(
+                hook_name,
+                hook_data_source.get('name'),
+                hook_data_source.get('hostname', 'localhost'),
+                hook_data_source.get('port'),
+            ),
+            restore_dump,
+            default_port,
+        )
     )
 
+    if not matching_dumps:
+        return None
 
-def get_configured_hook_name_and_database(hooks, database_name):
+    if len(matching_dumps) > 1:
+        raise ValueError(
+            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
+        )
+
+    return matching_dumps[0]
+
+
+def strip_path_prefix_from_extracted_dump_destination(
+    destination_path, borgmatic_runtime_directory
+):
     '''
-    Find the hook name and first database dict with the given database name in the configured hooks
-    dict. This searches across all database hooks.
+    Directory-format dump files get extracted into a temporary directory containing a path prefix
+    that depends how the files were stored in the archive. So, given the destination path where the
+    dump was extracted and the borgmatic runtime directory, move the dump files such that the
+    restore doesn't have to deal with that varying path prefix.
+
+    For instance, if the dump was extracted to:
+
+      /run/user/0/borgmatic/tmp1234/borgmatic/postgresql_databases/test/...
+
+    or:
+
+      /run/user/0/borgmatic/tmp1234/root/.borgmatic/postgresql_databases/test/...
+
+    then this function moves it to:
+
+      /run/user/0/borgmatic/postgresql_databases/test/...
     '''
+    for subdirectory_path, _, _ in os.walk(destination_path):
+        databases_directory = os.path.basename(subdirectory_path)
+
+        if not databases_directory.endswith('_databases'):
+            continue
+
+        shutil.move(
+            subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)
+        )
+        break
 
 
-def restore_single_database(
+def restore_single_dump(
     repository,
-    location,
-    storage,
-    hooks,
+    config,
     local_borg_version,
     global_arguments,
     local_path,
     remote_path,
     archive_name,
     hook_name,
-    database,
-):  # pragma: no cover
+    data_source,
+    connection_params,
+    borgmatic_runtime_directory,
+):
     '''
-    Given (among other things) an archive name, a database hook name, and a configured database
-    configuration dict, restore that database from the archive.
+    Given (among other things) an archive name, a data source hook name, the hostname, port,
+    username/password as connection params, and a configured data source configuration dict, restore
+    that data source from the archive.
     '''
-    logger.info(
-        f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}'
+    dump_metadata = render_dump_metadata(
+        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
     )
 
-    dump_pattern = borgmatic.hooks.dispatch.call_hooks(
-        'make_database_dump_pattern',
-        hooks,
-        repository['path'],
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        location,
-        database['name'],
-    )[hook_name]
+    logger.info(f'Restoring data source {dump_metadata}')
 
-    # Kick off a single database extract to stdout.
-    extract_process = borgmatic.borg.extract.extract_archive(
+    dump_patterns = borgmatic.hooks.dispatch.call_hooks(
+        'make_data_source_dump_patterns',
+        config,
+        borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+        borgmatic_runtime_directory,
+        data_source['name'],
+    )[hook_name.split('_databases', 1)[0]]
+
+    destination_path = (
+        tempfile.mkdtemp(dir=borgmatic_runtime_directory)
+        if data_source.get('format') == 'directory'
+        else None
+    )
+
+    try:
+        # Kick off a single data source extract. If using a directory format, extract to a temporary
+        # directory. Otherwise extract the single dump file to stdout.
+        extract_process = borgmatic.borg.extract.extract_archive(
+            dry_run=global_arguments.dry_run,
+            repository=repository['path'],
+            archive=archive_name,
+            paths=[
+                borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
+                    dump_patterns
+                )
+            ],
+            config=config,
+            local_borg_version=local_borg_version,
+            global_arguments=global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+            destination_path=destination_path,
+            # A directory format dump isn't a single file, and therefore can't extract
+            # to stdout. In this case, the extract_process return value is None.
+            extract_to_stdout=bool(data_source.get('format') != 'directory'),
+        )
+
+        if destination_path and not global_arguments.dry_run:
+            strip_path_prefix_from_extracted_dump_destination(
+                destination_path, borgmatic_runtime_directory
+            )
+    finally:
+        if destination_path and not global_arguments.dry_run:
+            shutil.rmtree(destination_path, ignore_errors=True)
+
+    # Run a single data source restore, consuming the extract stdout (if any).
+    borgmatic.hooks.dispatch.call_hook(
+        function_name='restore_data_source_dump',
+        config=config,
+        hook_name=hook_name,
+        data_source=data_source,
         dry_run=global_arguments.dry_run,
-        repository=repository['path'],
-        archive=archive_name,
-        paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
-        location_config=location,
-        storage_config=storage,
-        local_borg_version=local_borg_version,
-        global_arguments=global_arguments,
-        local_path=local_path,
-        remote_path=remote_path,
-        destination_path='/',
-        # A directory format dump isn't a single file, and therefore can't extract
-        # to stdout. In this case, the extract_process return value is None.
-        extract_to_stdout=bool(database.get('format') != 'directory'),
-    )
-
-    # Run a single database restore, consuming the extract stdout (if any).
-    borgmatic.hooks.dispatch.call_hooks(
-        'restore_database_dump',
-        {hook_name: [database]},
-        repository['path'],
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        location,
-        global_arguments.dry_run,
-        extract_process,
+        extract_process=extract_process,
+        connection_params=connection_params,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
     )
 
 
-def collect_archive_database_names(
+def collect_dumps_from_archive(
     repository,
     archive,
-    location,
-    storage,
+    config,
     local_borg_version,
     global_arguments,
     local_path,
     remote_path,
+    borgmatic_runtime_directory,
 ):
     '''
-    Given a local or remote repository path, a resolved archive name, a location configuration dict,
-    a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace,
-    and local and remote Borg paths, query the archive for the names of databases it contains and
-    return them as a dict from hook name to a sequence of database names.
+    Given a local or remote repository path, a resolved archive name, a configuration dict, the
+    local Borg version, global arguments an argparse.Namespace, local and remote Borg paths, and the
+    borgmatic runtime directory, query the archive for the names of data sources dumps it contains
+    and return them as a set of Dump instances.
     '''
-    borgmatic_source_directory = os.path.expanduser(
-        location.get(
-            'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
-        )
-    ).lstrip('/')
-    parent_dump_path = os.path.expanduser(
-        borgmatic.hooks.dump.make_database_dump_path(borgmatic_source_directory, '*_databases/*/*')
+    borgmatic_source_directory = str(
+        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
     )
+
+    # Probe for the data source dumps in multiple locations, as the default location has moved to
+    # the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
+    # we still want to support reading dumps from previously created archives as well.
     dump_paths = borgmatic.borg.list.capture_archive_listing(
         repository,
         archive,
-        storage,
+        config,
         local_borg_version,
         global_arguments,
-        list_path=parent_dump_path,
+        list_paths=[
+            'sh:'
+            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
+                base_directory, '*_databases/*/*'
+            )
+            for base_directory in (
+                'borgmatic',
+                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+                borgmatic_source_directory.lstrip('/'),
+            )
+        ],
         local_path=local_path,
         remote_path=remote_path,
     )
 
-    # Determine the database names corresponding to the dumps found in the archive and
-    # add them to restore_names.
-    archive_database_names = {}
+    # Parse the paths of dumps found in the archive to get their respective dump metadata.
+    dumps_from_archive = set()
 
     for dump_path in dump_paths:
-        try:
-            (hook_name, _, database_name) = dump_path.split(
-                borgmatic_source_directory + os.path.sep, 1
-            )[1].split(os.path.sep)[0:3]
-        except (ValueError, IndexError):
-            logger.warning(
-                f'{repository}: Ignoring invalid database dump path "{dump_path}" in archive {archive}'
-            )
+        if not dump_path:
+            continue
+
+        # Probe to find the base directory that's at the start of the dump path.
+        for base_directory in (
+            'borgmatic',
+            borgmatic_runtime_directory,
+            borgmatic_source_directory,
+        ):
+            try:
+                (hook_name, host_and_port, data_source_name) = dump_path.split(
+                    base_directory + os.path.sep, 1
+                )[1].split(os.path.sep)[0:3]
+            except (ValueError, IndexError):
+                continue
+
+            parts = host_and_port.split(':', 1)
+
+            if len(parts) == 1:
+                parts += (None,)
+
+            (hostname, port) = parts
+
+            try:
+                port = int(port)
+            except (ValueError, TypeError):
+                port = None
+
+            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
+
+            # We've successfully parsed the dump path, so need to probe any further.
+            break
         else:
-            if database_name not in archive_database_names.get(hook_name, []):
-                archive_database_names.setdefault(hook_name, []).extend([database_name])
+            logger.warning(
+                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
+            )
 
-    return archive_database_names
+    return dumps_from_archive
 
 
-def find_databases_to_restore(requested_database_names, archive_database_names):
+def get_dumps_to_restore(restore_arguments, dumps_from_archive):
     '''
-    Given a sequence of requested database names to restore and a dict of hook name to the names of
-    databases found in an archive, return an expanded sequence of database names to restore,
-    replacing "all" with actual database names as appropriate.
+    Given restore arguments as an argparse.Namespace instance indicating which dumps to restore and
+    a set of Dump instances representing the dumps found in an archive, return a set of specific
+    Dump instances from the archive to restore. As part of this, replace any Dump having a data
+    source name of "all" with multiple named Dump instances as appropriate.
 
-    Raise ValueError if any of the requested database names cannot be found in the archive.
+    Raise ValueError if any of the requested data source names cannot be found in the archive or if
+    there are multiple archive dump matches for a given requested dump.
     '''
-    # A map from database hook name to the database names to restore for that hook.
-    restore_names = (
-        {UNSPECIFIED_HOOK: requested_database_names}
-        if requested_database_names
-        else {UNSPECIFIED_HOOK: ['all']}
+    requested_dumps = (
+        {
+            Dump(
+                hook_name=(
+                    (
+                        restore_arguments.hook
+                        if restore_arguments.hook.endswith('_databases')
+                        else f'{restore_arguments.hook}_databases'
+                    )
+                    if restore_arguments.hook
+                    else UNSPECIFIED
+                ),
+                data_source_name=name,
+                hostname=restore_arguments.original_hostname or UNSPECIFIED,
+                port=restore_arguments.original_port,
+            )
+            for name in restore_arguments.data_sources or (UNSPECIFIED,)
+        }
+        if restore_arguments.hook
+        or restore_arguments.data_sources
+        or restore_arguments.original_hostname
+        or restore_arguments.original_port
+        else {
+            Dump(
+                hook_name=UNSPECIFIED,
+                data_source_name='all',
+                hostname=UNSPECIFIED,
+                port=UNSPECIFIED,
+            )
+        }
     )
+    missing_dumps = set()
+    dumps_to_restore = set()
 
-    # If "all" is in restore_names, then replace it with the names of dumps found within the
-    # archive.
-    if 'all' in restore_names[UNSPECIFIED_HOOK]:
-        restore_names[UNSPECIFIED_HOOK].remove('all')
+    # If there's a requested "all" dump, add every dump from the archive to the dumps to restore.
+    if any(dump for dump in requested_dumps if dump.data_source_name == 'all'):
+        dumps_to_restore.update(dumps_from_archive)
 
-        for hook_name, database_names in archive_database_names.items():
-            restore_names.setdefault(hook_name, []).extend(database_names)
+    # If any archive dump matches a requested dump, add the archive dump to the dumps to restore.
+    for requested_dump in requested_dumps:
+        if requested_dump.data_source_name == 'all':
+            continue
 
-            # If a database is to be restored as part of "all", then remove it from restore names so
-            # it doesn't get restored twice.
-            for database_name in database_names:
-                if database_name in restore_names[UNSPECIFIED_HOOK]:
-                    restore_names[UNSPECIFIED_HOOK].remove(database_name)
-
-    if not restore_names[UNSPECIFIED_HOOK]:
-        restore_names.pop(UNSPECIFIED_HOOK)
-
-    combined_restore_names = set(
-        name for database_names in restore_names.values() for name in database_names
-    )
-    combined_archive_database_names = set(
-        name for database_names in archive_database_names.values() for name in database_names
-    )
-
-    missing_names = sorted(set(combined_restore_names) - combined_archive_database_names)
-    if missing_names:
-        joined_names = ', '.join(f'"{name}"' for name in missing_names)
-        raise ValueError(
-            f"Cannot restore database{'s' if len(missing_names) > 1 else ''} {joined_names} missing from archive"
+        matching_dumps = tuple(
+            archive_dump
+            for archive_dump in dumps_from_archive
+            if dumps_match(requested_dump, archive_dump)
         )
 
-    return restore_names
+        if len(matching_dumps) == 0:
+            missing_dumps.add(requested_dump)
+        elif len(matching_dumps) == 1:
+            dumps_to_restore.add(matching_dumps[0])
+        else:
+            raise ValueError(
+                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
+            )
+
+    if missing_dumps:
+        rendered_dumps = ', '.join(
+            f'{render_dump_metadata(dump)}' for dump in sorted(missing_dumps)
+        )
+
+        raise ValueError(
+            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
+        )
+
+    return dumps_to_restore
 
 
-def ensure_databases_found(restore_names, remaining_restore_names, found_names):
+def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
     '''
-    Given a dict from hook name to database names to restore, a dict from hook name to remaining
-    database names to restore, and a sequence of found (actually restored) database names, raise
-    ValueError if requested databases to restore were missing from the archive and/or configuration.
+    Given a set of requested dumps to restore and a set of dumps actually restored, raise ValueError
+    if any requested dumps to restore weren't restored, indicating that they were missing from the
+    configuration.
     '''
-    combined_restore_names = set(
-        name
-        for database_names in tuple(restore_names.values())
-        + tuple(remaining_restore_names.values())
-        for name in database_names
+    if not dumps_actually_restored:
+        raise ValueError('No data source dumps were found to restore')
+
+    missing_dumps = sorted(
+        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
     )
 
-    if not combined_restore_names and not found_names:
-        raise ValueError('No databases were found to restore')
+    if missing_dumps:
+        rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
 
-    missing_names = sorted(set(combined_restore_names) - set(found_names))
-    if missing_names:
-        joined_names = ', '.join(f'"{name}"' for name in missing_names)
         raise ValueError(
-            f"Cannot restore database{'s' if len(missing_names) > 1 else ''} {joined_names} missing from borgmatic's configuration"
+            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
         )
 
 
 def run_restore(
     repository,
-    location,
-    storage,
-    hooks,
+    config,
     local_borg_version,
     restore_arguments,
     global_arguments,
@@ -259,112 +434,98 @@ def run_restore(
     Run the "restore" action for the given repository, but only if the repository matches the
     requested repository in restore arguments.
 
-    Raise ValueError if a configured database could not be found to restore.
+    Raise ValueError if a configured data source could not be found to restore or there's no
+    matching dump in the archive.
     '''
     if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
         repository, restore_arguments.repository
     ):
         return
 
-    logger.info(
-        f'{repository.get("label", repository["path"])}: Restoring databases from archive {restore_arguments.archive}'
-    )
+    logger.info(f'Restoring data sources from archive {restore_arguments.archive}')
 
-    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-        'remove_database_dumps',
-        hooks,
-        repository['path'],
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        location,
-        global_arguments.dry_run,
-    )
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            'remove_data_source_dumps',
+            config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            borgmatic_runtime_directory,
+            global_arguments.dry_run,
+        )
 
-    archive_name = borgmatic.borg.rlist.resolve_archive_name(
-        repository['path'],
-        restore_arguments.archive,
-        storage,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-    archive_database_names = collect_archive_database_names(
-        repository['path'],
-        archive_name,
-        location,
-        storage,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-    restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
-    found_names = set()
-    remaining_restore_names = {}
+        archive_name = borgmatic.borg.repo_list.resolve_archive_name(
+            repository['path'],
+            restore_arguments.archive,
+            config,
+            local_borg_version,
+            global_arguments,
+            local_path,
+            remote_path,
+        )
+        dumps_from_archive = collect_dumps_from_archive(
+            repository['path'],
+            archive_name,
+            config,
+            local_borg_version,
+            global_arguments,
+            local_path,
+            remote_path,
+            borgmatic_runtime_directory,
+        )
+        dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
 
-    for hook_name, database_names in restore_names.items():
-        for database_name in database_names:
-            found_hook_name, found_database = get_configured_database(
-                hooks, archive_database_names, hook_name, database_name
+        dumps_actually_restored = set()
+        connection_params = {
+            'hostname': restore_arguments.hostname,
+            'port': restore_arguments.port,
+            'username': restore_arguments.username,
+            'password': restore_arguments.password,
+            'restore_path': restore_arguments.restore_path,
+        }
+
+        # Restore each dump.
+        for restore_dump in dumps_to_restore:
+            found_data_source = get_configured_data_source(
+                config,
+                restore_dump,
             )
 
-            if not found_database:
-                remaining_restore_names.setdefault(found_hook_name or hook_name, []).append(
-                    database_name
+            # For a dump that wasn't found via an exact match in the configuration, try to fallback
+            # to an "all" data source.
+            if not found_data_source:
+                found_data_source = get_configured_data_source(
+                    config,
+                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
                 )
-                continue
 
-            found_names.add(database_name)
-            restore_single_database(
+                if not found_data_source:
+                    continue
+
+                found_data_source = dict(found_data_source)
+                found_data_source['name'] = restore_dump.data_source_name
+
+            dumps_actually_restored.add(restore_dump)
+
+            restore_single_dump(
                 repository,
-                location,
-                storage,
-                hooks,
+                config,
                 local_borg_version,
                 global_arguments,
                 local_path,
                 remote_path,
                 archive_name,
-                found_hook_name or hook_name,
-                dict(found_database, **{'schemas': restore_arguments.schemas}),
+                restore_dump.hook_name,
+                dict(found_data_source, **{'schemas': restore_arguments.schemas}),
+                connection_params,
+                borgmatic_runtime_directory,
             )
 
-    # For any database that weren't found via exact matches in the hooks configuration, try to
-    # fallback to "all" entries.
-    for hook_name, database_names in remaining_restore_names.items():
-        for database_name in database_names:
-            found_hook_name, found_database = get_configured_database(
-                hooks, archive_database_names, hook_name, database_name, 'all'
-            )
+        borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
+            'remove_data_source_dumps',
+            config,
+            borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
+            borgmatic_runtime_directory,
+            global_arguments.dry_run,
+        )
 
-            if not found_database:
-                continue
-
-            found_names.add(database_name)
-            database = copy.copy(found_database)
-            database['name'] = database_name
-
-            restore_single_database(
-                repository,
-                location,
-                storage,
-                hooks,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-                archive_name,
-                found_hook_name or hook_name,
-                dict(database, **{'schemas': restore_arguments.schemas}),
-            )
-
-    borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
-        'remove_database_dumps',
-        hooks,
-        repository['path'],
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        location,
-        global_arguments.dry_run,
-    )
-
-    ensure_databases_found(restore_names, remaining_restore_names, found_names)
+    ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored)

borgmatic/actions/rinfo.py

@@ -1,42 +0,0 @@
-import json
-import logging
-
-import borgmatic.borg.rinfo
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_rinfo(
-    repository,
-    storage,
-    local_borg_version,
-    rinfo_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "rinfo" action for the given repository.
-
-    If rinfo_arguments.json is True, yield the JSON output from the info for the repository.
-    '''
-    if rinfo_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, rinfo_arguments.repository
-    ):
-        if not rinfo_arguments.json:  # pragma: nocover
-            logger.answer(
-                f'{repository.get("label", repository["path"])}: Displaying repository summary information'
-            )
-
-        json_output = borgmatic.borg.rinfo.display_repository_info(
-            repository['path'],
-            storage,
-            local_borg_version,
-            rinfo_arguments=rinfo_arguments,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        if json_output:  # pragma: nocover
-            yield json.loads(json_output)

borgmatic/actions/rlist.py

@@ -1,40 +0,0 @@
-import json
-import logging
-
-import borgmatic.borg.rlist
-import borgmatic.config.validate
-
-logger = logging.getLogger(__name__)
-
-
-def run_rlist(
-    repository,
-    storage,
-    local_borg_version,
-    rlist_arguments,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Run the "rlist" action for the given repository.
-
-    If rlist_arguments.json is True, yield the JSON output from listing the repository.
-    '''
-    if rlist_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, rlist_arguments.repository
-    ):
-        if not rlist_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository.get("label", repository["path"])}: Listing repository')
-
-        json_output = borgmatic.borg.rlist.list_repository(
-            repository['path'],
-            storage,
-            local_borg_version,
-            rlist_arguments=rlist_arguments,
-            global_arguments=global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        if json_output:  # pragma: nocover
-            yield json.loads(json_output)

borgmatic/actions/transfer.py

@@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
 
 def run_transfer(
     repository,
-    storage,
+    config,
     local_borg_version,
     transfer_arguments,
     global_arguments,
@@ -17,13 +17,11 @@ def run_transfer(
     '''
     Run the "transfer" action for the given repository.
     '''
-    logger.info(
-        f'{repository.get("label", repository["path"])}: Transferring archives to repository'
-    )
+    logger.info('Transferring archives to repository')
     borgmatic.borg.transfer.transfer_archives(
         global_arguments.dry_run,
         repository['path'],
-        storage,
+        config,
         local_borg_version,
         transfer_arguments,
         global_arguments,

borgmatic/borg/borg.py

@@ -1,6 +1,8 @@
 import logging
+import shlex
 
 import borgmatic.commands.arguments
+import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@@ -8,14 +10,12 @@ from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 logger = logging.getLogger(__name__)
 
 
-REPOSITORYLESS_BORG_COMMANDS = {'serve', None}
 BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'}
-BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-profile'), ())
 
 
 def run_arbitrary_borg(
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
     options,
     archive=None,
@@ -23,12 +23,13 @@ def run_arbitrary_borg(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, a
+    Given a local or remote repository path, a configuration dict, the local Borg version, a
     sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary
-    Borg command on the given repository/archive.
+    Borg command, passing in REPOSITORY and ARCHIVE environment variables for optional use in the
+    command.
     '''
     borgmatic.logger.add_custom_log_levels()
-    lock_wait = storage_config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait', None)
 
     try:
         options = options[1:] if options[0] == '--' else options
@@ -38,10 +39,7 @@ def run_arbitrary_borg(
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
 
-        if (
-            borg_command
-            and borg_command[0] in borgmatic.commands.arguments.SUBPARSER_ALIASES.keys()
-        ):
+        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
             logger.warning(
                 f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
             )
@@ -49,29 +47,28 @@ def run_arbitrary_borg(
         borg_command = ()
         command_options = ()
 
-    if borg_command in BORG_SUBCOMMANDS_WITHOUT_REPOSITORY:
-        repository_archive_flags = ()
-    elif archive:
-        repository_archive_flags = flags.make_repository_archive_flags(
-            repository_path, archive, local_borg_version
-        )
-    else:
-        repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version)
-
     full_command = (
         (local_path,)
         + borg_command
-        + repository_archive_flags
-        + command_options
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
         + flags.make_flags('lock-wait', lock_wait)
+        + command_options
     )
 
     return execute_command(
-        full_command,
+        tuple(shlex.quote(part) for part in full_command),
         output_file=DO_NOT_CAPTURE,
+        shell=True,
+        environment=dict(
+            (environment.make_environment(config) or {}),
+            **{
+                'BORG_REPO': repository_path,
+                'ARCHIVE': archive if archive else '',
+            },
+        ),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
+        borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/break_lock.py

@@ -1,5 +1,6 @@
 import logging
 
+import borgmatic.config.paths
 from borgmatic.borg import environment, flags
 from borgmatic.execute import execute_command
 
@@ -8,19 +9,19 @@ logger = logging.getLogger(__name__)
 
 def break_lock(
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
     global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage configuration dict, the local Borg version,
-    an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
+    Given a local or remote repository path, a configuration dict, the local Borg version, an
+    argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
     repository and cache locks leftover from Borg aborting.
     '''
-    umask = storage_config.get('umask', None)
-    lock_wait = storage_config.get('lock_wait', None)
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
 
     full_command = (
         (local_path, 'break-lock')
@@ -33,5 +34,10 @@ def break_lock(
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    borg_environment = environment.make_environment(storage_config)
-    execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment)
+    execute_command(
+        full_command,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/change_passphrase.py

@@ -0,0 +1,67 @@
+import logging
+
+import borgmatic.config.paths
+import borgmatic.execute
+import borgmatic.logger
+from borgmatic.borg import environment, flags
+
+logger = logging.getLogger(__name__)
+
+
+def change_passphrase(
+    repository_path,
+    config,
+    local_borg_version,
+    change_passphrase_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, change
+    passphrase arguments, and optional local and remote Borg paths, change the repository passphrase
+    based on an interactive prompt.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'key', 'change-passphrase')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + flags.make_repository_flags(
+            repository_path,
+            local_borg_version,
+        )
+    )
+
+    if global_arguments.dry_run:
+        logger.info('Skipping change password (dry run)')
+        return
+
+    # If the original passphrase is set programmatically, then Borg won't prompt for a new one! So
+    # don't give Borg any passphrase, and it'll ask the user for both old and new ones.
+    config_without_passphrase = {
+        option_name: value
+        for (option_name, value) in config.items()
+        if option_name not in ('encryption_passphrase', 'encryption_passcommand')
+    }
+
+    borgmatic.execute.execute_command(
+        full_command,
+        output_file=borgmatic.execute.DO_NOT_CAPTURE,
+        output_log_level=logging.ANSWER,
+        environment=environment.make_environment(config_without_passphrase),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )
+
+    logger.answer(
+        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)"
+    )

borgmatic/borg/check.py

@@ -1,169 +1,27 @@
 import argparse
-import datetime
-import hashlib
-import itertools
 import json
 import logging
-import os
-import pathlib
 
-from borgmatic.borg import environment, extract, feature, flags, rinfo, state
+import borgmatic.config.paths
+from borgmatic.borg import environment, feature, flags, repo_info
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
-DEFAULT_CHECKS = (
-    {'name': 'repository', 'frequency': '1 month'},
-    {'name': 'archives', 'frequency': '1 month'},
-)
-
-
 logger = logging.getLogger(__name__)
 
 
-def parse_checks(consistency_config, only_checks=None):
+def make_archive_filter_flags(local_borg_version, config, checks, check_arguments):
     '''
-    Given a consistency config with a "checks" sequence of dicts and an optional list of override
-    checks, return a tuple of named checks to run.
+    Given the local Borg version, a configuration dict, a parsed sequence of checks, and check
+    arguments as an argparse.Namespace instance, transform the checks into tuple of command-line
+    flags for filtering archives in a check command.
 
-    For example, given a retention config of:
-
-        {'checks': ({'name': 'repository'}, {'name': 'archives'})}
-
-    This will be returned as:
-
-        ('repository', 'archives')
-
-    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If a checks value
-    has a name of "disabled", return an empty tuple, meaning that no checks should be run.
+    If "check_last" is set in the configuration and "archives" is in checks, then include a "--last"
+    flag. And if "prefix" is set in configuration and "archives" is in checks, then include a
+    "--match-archives" flag.
     '''
-    checks = only_checks or tuple(
-        check_config['name']
-        for check_config in (consistency_config.get('checks', None) or DEFAULT_CHECKS)
-    )
-    checks = tuple(check.lower() for check in checks)
-    if 'disabled' in checks:
-        if len(checks) > 1:
-            logger.warning(
-                'Multiple checks are configured, but one of them is "disabled"; not running any checks'
-            )
-        return ()
+    check_last = config.get('check_last', None)
+    prefix = config.get('prefix')
 
-    return checks
-
-
-def parse_frequency(frequency):
-    '''
-    Given a frequency string with a number and a unit of time, return a corresponding
-    datetime.timedelta instance or None if the frequency is None or "always".
-
-    For instance, given "3 weeks", return datetime.timedelta(weeks=3)
-
-    Raise ValueError if the given frequency cannot be parsed.
-    '''
-    if not frequency:
-        return None
-
-    frequency = frequency.strip().lower()
-
-    if frequency == 'always':
-        return None
-
-    try:
-        number, time_unit = frequency.split(' ')
-        number = int(number)
-    except ValueError:
-        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
-
-    if not time_unit.endswith('s'):
-        time_unit += 's'
-
-    if time_unit == 'months':
-        number *= 30
-        time_unit = 'days'
-    elif time_unit == 'years':
-        number *= 365
-        time_unit = 'days'
-
-    try:
-        return datetime.timedelta(**{time_unit: number})
-    except TypeError:
-        raise ValueError(f"Could not parse consistency check frequency '{frequency}'")
-
-
-def filter_checks_on_frequency(
-    location_config,
-    consistency_config,
-    borg_repository_id,
-    checks,
-    force,
-    archives_check_id=None,
-):
-    '''
-    Given a location config, a consistency config with a "checks" sequence of dicts, a Borg
-    repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives
-    check potentially being run (if any), filter down those checks based on the configured
-    "frequency" for each check as compared to its check time file.
-
-    In other words, a check whose check time file's timestamp is too new (based on the configured
-    frequency) will get cut from the returned sequence of checks. Example:
-
-    consistency_config = {
-        'checks': [
-            {
-                'name': 'archives',
-                'frequency': '2 weeks',
-            },
-        ]
-    }
-
-    When this function is called with that consistency_config and "archives" in checks, "archives"
-    will get filtered out of the returned result if its check time file is newer than 2 weeks old,
-    indicating that it's not yet time to run that check again.
-
-    Raise ValueError if a frequency cannot be parsed.
-    '''
-    filtered_checks = list(checks)
-
-    if force:
-        return tuple(filtered_checks)
-
-    for check_config in consistency_config.get('checks', DEFAULT_CHECKS):
-        check = check_config['name']
-        if checks and check not in checks:
-            continue
-
-        frequency_delta = parse_frequency(check_config.get('frequency'))
-        if not frequency_delta:
-            continue
-
-        check_time = probe_for_check_time(
-            location_config, borg_repository_id, check, archives_check_id
-        )
-        if not check_time:
-            continue
-
-        # If we've not yet reached the time when the frequency dictates we're ready for another
-        # check, skip this check.
-        if datetime.datetime.now() < check_time + frequency_delta:
-            remaining = check_time + frequency_delta - datetime.datetime.now()
-            logger.info(
-                f'Skipping {check} check due to configured frequency; {remaining} until next check'
-            )
-            filtered_checks.remove(check)
-
-    return tuple(filtered_checks)
-
-
-def make_archive_filter_flags(
-    local_borg_version, storage_config, checks, check_last=None, prefix=None
-):
-    '''
-    Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the
-    check last value, and a consistency check prefix, transform the checks into tuple of
-    command-line flags for filtering archives in a check command.
-
-    If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if
-    a prefix value is given and "archives" is in checks, then include a "--match-archives" flag.
-    '''
     if 'archives' in checks or 'data' in checks:
         return (('--last', str(check_last)) if check_last else ()) + (
             (
@@ -174,8 +32,8 @@ def make_archive_filter_flags(
             if prefix
             else (
                 flags.make_match_archives_flags(
-                    storage_config.get('match_archives'),
-                    storage_config.get('archive_name_format'),
+                    check_arguments.match_archives or config.get('match_archives'),
+                    config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
@@ -193,21 +51,10 @@ def make_archive_filter_flags(
     return ()
 
 
-def make_archives_check_id(archive_filter_flags):
+def make_check_name_flags(checks, archive_filter_flags):
     '''
-    Given a sequence of flags to filter archives, return a unique hash corresponding to those
-    particular flags. If there are no flags, return None.
-    '''
-    if not archive_filter_flags:
-        return None
-
-    return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
-
-
-def make_check_flags(checks, archive_filter_flags):
-    '''
-    Given a parsed sequence of checks and a sequence of flags to filter archives, transform the
-    checks into tuple of command-line check flags.
+    Given parsed checks set and a sequence of flags to filter archives, transform the checks into
+    tuple of command-line check flags.
 
     For example, given parsed checks of:
 
@@ -217,18 +64,14 @@ def make_check_flags(checks, archive_filter_flags):
 
         ('--repository-only',)
 
-    However, if both "repository" and "archives" are in checks, then omit them from the returned
-    flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
+    However, if both "repository" and "archives" are in checks, then omit the "only" flags from the
+    returned flags because Borg does both checks by default. Note that a "data" check only works
+    along with an "archives" check.
     '''
-    if 'data' in checks:
-        data_flags = ('--verify-data',)
-        checks += ('archives',)
-    else:
-        data_flags = ()
-
+    data_flags = ('--verify-data',) if 'data' in checks else ()
     common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
 
-    if {'repository', 'archives'}.issubset(set(checks)):
+    if {'repository', 'archives'}.issubset(checks):
         return common_flags
 
     return (
@@ -237,153 +80,20 @@ def make_check_flags(checks, archive_filter_flags):
     )
 
 
-def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None):
-    '''
-    Given a location configuration dict, a Borg repository ID, the name of a check type
-    ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
-    path for recording that check's time (the time of that check last occurring).
-    '''
-    borgmatic_source_directory = os.path.expanduser(
-        location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY)
-    )
-
-    if check_type in ('archives', 'data'):
-        return os.path.join(
-            borgmatic_source_directory,
-            'checks',
-            borg_repository_id,
-            check_type,
-            archives_check_id if archives_check_id else 'all',
-        )
-
-    return os.path.join(
-        borgmatic_source_directory,
-        'checks',
-        borg_repository_id,
-        check_type,
-    )
-
-
-def write_check_time(path):  # pragma: no cover
-    '''
-    Record a check time of now as the modification time of the given path.
-    '''
-    logger.debug(f'Writing check time at {path}')
-
-    os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-    pathlib.Path(path, mode=0o600).touch()
-
-
-def read_check_time(path):
-    '''
-    Return the check time based on the modification time of the given path. Return None if the path
-    doesn't exist.
-    '''
-    logger.debug(f'Reading check time from {path}')
-
-    try:
-        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
-    except FileNotFoundError:
-        return None
-
-
-def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id):
-    '''
-    Given a location configuration dict, a Borg repository ID, the name of a check type
-    ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
-    the corresponding check time or None if such a check time does not exist.
-
-    When the check type is "archives" or "data", this function probes two different paths to find
-    the check time, e.g.:
-
-      ~/.borgmatic/checks/1234567890/archives/9876543210
-      ~/.borgmatic/checks/1234567890/archives/all
-
-    ... and returns the maximum modification time of the files found (if any). The first path
-    represents a more specific archives check time (a check on a subset of archives), and the second
-    is a fallback to the last "all" archives check.
-
-    For other check types, this function reads from a single check time path, e.g.:
-
-      ~/.borgmatic/checks/1234567890/repository
-    '''
-    check_times = (
-        read_check_time(group[0])
-        for group in itertools.groupby(
-            (
-                make_check_time_path(location_config, borg_repository_id, check, archives_check_id),
-                make_check_time_path(location_config, borg_repository_id, check),
-            )
-        )
-    )
-
-    try:
-        return max(check_time for check_time in check_times if check_time)
-    except ValueError:
-        return None
-
-
-def upgrade_check_times(location_config, borg_repository_id):
-    '''
-    Given a location configuration dict and a Borg repository ID, upgrade any corresponding check
-    times on disk from old-style paths to new-style paths.
-
-    Currently, the only upgrade performed is renaming an archive or data check path that looks like:
-
-      ~/.borgmatic/checks/1234567890/archives
-
-    to:
-
-      ~/.borgmatic/checks/1234567890/archives/all
-    '''
-    for check_type in ('archives', 'data'):
-        new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all')
-        old_path = os.path.dirname(new_path)
-        temporary_path = f'{old_path}.temp'
-
-        if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
-            return
-
-        logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
-
-        try:
-            os.rename(old_path, temporary_path)
-        except FileNotFoundError:
-            pass
-
-        os.mkdir(old_path)
-        os.rename(temporary_path, new_path)
-
-
-def check_archives(
-    repository_path,
-    location_config,
-    storage_config,
-    consistency_config,
-    local_borg_version,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-    progress=None,
-    repair=None,
-    only_checks=None,
-    force=None,
+def get_repository_id(
+    repository_path, config, local_borg_version, global_arguments, local_path, remote_path
 ):
     '''
-    Given a local or remote repository path, a storage config dict, a consistency config dict,
-    local/remote commands to run, whether to include progress information, whether to attempt a
-    repair, and an optional list of checks to use instead of configured checks, check the contained
-    Borg archives for consistency.
+    Given a local or remote repository path, a configuration dict, the local Borg version, global
+    arguments, and local/remote commands to run, return the corresponding Borg repository ID.
 
-    If there are no consistency checks to run, skip running them.
-
-    Raises ValueError if the Borg repository ID cannot be determined.
+    Raise ValueError if the Borg repository ID cannot be determined.
     '''
     try:
-        borg_repository_id = json.loads(
-            rinfo.display_repository_info(
+        return json.loads(
+            repo_info.display_repository_info(
                 repository_path,
-                storage_config,
+                config,
                 local_borg_version,
                 argparse.Namespace(json=True),
                 global_arguments,
@@ -394,73 +104,86 @@ def check_archives(
     except (json.JSONDecodeError, KeyError):
         raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
 
-    upgrade_check_times(location_config, borg_repository_id)
 
-    check_last = consistency_config.get('check_last', None)
-    prefix = consistency_config.get('prefix')
-    configured_checks = parse_checks(consistency_config, only_checks)
-    lock_wait = None
-    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
-    archive_filter_flags = make_archive_filter_flags(
-        local_borg_version, storage_config, configured_checks, check_last, prefix
-    )
-    archives_check_id = make_archives_check_id(archive_filter_flags)
+def check_archives(
+    repository_path,
+    config,
+    local_borg_version,
+    check_arguments,
+    global_arguments,
+    checks,
+    archive_filter_flags,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, check
+    arguments as an argparse.Namespace instance, global arguments, a set of named Borg checks to run
+    (some combination "repository", "archives", and/or "data"), archive filter flags, and
+    local/remote commands to run, check the contained Borg archives for consistency.
+    '''
+    lock_wait = config.get('lock_wait')
+    extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
 
-    checks = filter_checks_on_frequency(
-        location_config,
-        consistency_config,
-        borg_repository_id,
-        configured_checks,
-        force,
-        archives_check_id,
-    )
+    verbosity_flags = ()
+    if logger.isEnabledFor(logging.INFO):
+        verbosity_flags = ('--info',)
+    if logger.isEnabledFor(logging.DEBUG):
+        verbosity_flags = ('--debug', '--show-rc')
 
-    if set(checks).intersection({'repository', 'archives', 'data'}):
-        lock_wait = storage_config.get('lock_wait')
+    try:
+        repository_check_config = next(
+            check for check in config.get('checks', ()) if check.get('name') == 'repository'
+        )
+    except StopIteration:
+        repository_check_config = {}
 
-        verbosity_flags = ()
-        if logger.isEnabledFor(logging.INFO):
-            verbosity_flags = ('--info',)
-        if logger.isEnabledFor(logging.DEBUG):
-            verbosity_flags = ('--debug', '--show-rc')
+    max_duration = check_arguments.max_duration or repository_check_config.get('max_duration')
 
+    umask = config.get('umask')
+    borg_exit_codes = config.get('borg_exit_codes')
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    if 'data' in checks:
+        checks.add('archives')
+
+    grouped_checks = (checks,)
+
+    # If max_duration is set, then archives and repository checks need to be run separately, as Borg
+    # doesn't support --max-duration along with an archives checks.
+    if max_duration and 'archives' in checks and 'repository' in checks:
+        checks.remove('repository')
+        grouped_checks = (checks, {'repository'})
+
+    for checks_subset in grouped_checks:
         full_command = (
             (local_path, 'check')
-            + (('--repair',) if repair else ())
-            + make_check_flags(checks, archive_filter_flags)
+            + (('--repair',) if check_arguments.repair else ())
+            + (
+                ('--max-duration', str(max_duration))
+                if max_duration and 'repository' in checks_subset
+                else ()
+            )
+            + make_check_name_flags(checks_subset, archive_filter_flags)
             + (('--remote-path', remote_path) if remote_path else ())
+            + (('--umask', str(umask)) if umask else ())
             + (('--log-json',) if global_arguments.log_json else ())
             + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
-            + (('--progress',) if progress else ())
+            + (('--progress',) if check_arguments.progress else ())
             + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
             + flags.make_repository_flags(repository_path, local_borg_version)
         )
 
-        borg_environment = environment.make_environment(storage_config)
-
-        # The Borg repair option triggers an interactive prompt, which won't work when output is
-        # captured. And progress messes with the terminal directly.
-        if repair or progress:
-            execute_command(
-                full_command, output_file=DO_NOT_CAPTURE, extra_environment=borg_environment
-            )
-        else:
-            execute_command(full_command, extra_environment=borg_environment)
-
-        for check in checks:
-            write_check_time(
-                make_check_time_path(location_config, borg_repository_id, check, archives_check_id)
-            )
-
-    if 'extract' in checks:
-        extract.extract_last_archive_dry_run(
-            storage_config,
-            local_borg_version,
-            global_arguments,
-            repository_path,
-            lock_wait,
-            local_path,
-            remote_path,
+        execute_command(
+            full_command,
+            # The Borg repair option triggers an interactive prompt, which won't work when output is
+            # captured. And progress messes with the terminal directly.
+            output_file=(
+                DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None
+            ),
+            environment=environment.make_environment(config),
+            working_directory=working_directory,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
         )
-        write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract'))

borgmatic/borg/compact.py

@@ -1,5 +1,6 @@
 import logging
 
+import borgmatic.config.paths
 from borgmatic.borg import environment, flags
 from borgmatic.execute import execute_command
 
@@ -9,7 +10,7 @@ logger = logging.getLogger(__name__)
 def compact_segments(
     dry_run,
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
     global_arguments,
     local_path='borg',
@@ -19,12 +20,12 @@ def compact_segments(
     threshold=None,
 ):
     '''
-    Given dry-run flag, a local or remote repository path, a storage config dict, and the local
-    Borg version, compact the segments in a repository.
+    Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg
+    version, compact the segments in a repository.
     '''
-    umask = storage_config.get('umask', None)
-    lock_wait = storage_config.get('lock_wait', None)
-    extra_borg_options = storage_config.get('extra_borg_options', {}).get('compact', '')
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('compact', '')
 
     full_command = (
         (local_path, 'compact')
@@ -42,12 +43,14 @@ def compact_segments(
     )
 
     if dry_run:
-        logging.info(f'{repository_path}: Skipping compact (dry run)')
+        logging.info('Skipping compact (dry run)')
         return
 
     execute_command(
         full_command,
         output_log_level=logging.INFO,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
+        borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/create.py

@@ -1,13 +1,15 @@
-import glob
 import itertools
 import logging
 import os
 import pathlib
 import stat
 import tempfile
+import textwrap
 
+import borgmatic.borg.pattern
+import borgmatic.config.paths
 import borgmatic.logger
-from borgmatic.borg import environment, feature, flags, state
+from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import (
     DO_NOT_CAPTURE,
     execute_command,
@@ -18,182 +20,53 @@ from borgmatic.execute import (
 logger = logging.getLogger(__name__)
 
 
-def expand_directory(directory):
+def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
     '''
-    Given a directory path, expand any tilde (representing a user's home directory) and any globs
-    therein. Return a list of one or more resulting paths.
-    '''
-    expanded_directory = os.path.expanduser(directory)
+    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
+    temporary file in the given borgmatic runtime directory and return the file object so it can
+    continue to exist on disk as long as the caller needs it.
 
-    return glob.glob(expanded_directory) or [expanded_directory]
-
-
-def expand_directories(directories):
-    '''
-    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
-    resulting directories as a single flattened tuple.
-    '''
-    if directories is None:
-        return ()
-
-    return tuple(
-        itertools.chain.from_iterable(expand_directory(directory) for directory in directories)
-    )
-
-
-def expand_home_directories(directories):
-    '''
-    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
-    Return the results as a tuple.
-    '''
-    if directories is None:
-        return ()
-
-    return tuple(os.path.expanduser(directory) for directory in directories)
-
-
-def map_directories_to_devices(directories):
-    '''
-    Given a sequence of directories, return a map from directory to an identifier for the device on
-    which that directory resides or None if the path doesn't exist.
-
-    This is handy for determining whether two different directories are on the same filesystem (have
-    the same device identifier).
-    '''
-    return {
-        directory: os.stat(directory).st_dev if os.path.exists(directory) else None
-        for directory in directories
-    }
-
-
-def deduplicate_directories(directory_devices, additional_directory_devices):
-    '''
-    Given a map from directory to the identifier for the device on which that directory resides,
-    return the directories as a sorted tuple with all duplicate child directories removed. For
-    instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
-
-    The one exception to this rule is if two paths are on different filesystems (devices). In that
-    case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
-    location.one_file_system option is true).
-
-    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
-    child directories, because it will naturally spider the contents of the parent directory. And
-    there are cases where Borg coming across the same file twice will result in duplicate reads and
-    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
-    Borg.
-
-    If any additional directory devices are given, also deduplicate against them, but don't include
-    them in the returned directories.
-    '''
-    deduplicated = set()
-    directories = sorted(directory_devices.keys())
-    additional_directories = sorted(additional_directory_devices.keys())
-    all_devices = {**directory_devices, **additional_directory_devices}
-
-    for directory in directories:
-        deduplicated.add(directory)
-        parents = pathlib.PurePath(directory).parents
-
-        # If another directory in the given list (or the additional list) is a parent of current
-        # directory (even n levels up) and both are on the same filesystem, then the current
-        # directory is a duplicate.
-        for other_directory in directories + additional_directories:
-            for parent in parents:
-                if (
-                    pathlib.PurePath(other_directory) == parent
-                    and all_devices[directory] is not None
-                    and all_devices[other_directory] == all_devices[directory]
-                ):
-                    if directory in deduplicated:
-                        deduplicated.remove(directory)
-                    break
-
-    return tuple(sorted(deduplicated))
-
-
-def write_pattern_file(patterns=None, sources=None, pattern_file=None):
-    '''
-    Given a sequence of patterns and an optional sequence of source directories, write them to a
-    named temporary file (with the source directories as additional roots) and return the file.
-    If an optional open pattern file is given, overwrite it instead of making a new temporary file.
+    If an optional open pattern file is given, append to it instead of making a new temporary file.
     Return None if no patterns are provided.
     '''
-    if not patterns and not sources:
+    if not patterns:
         return None
 
-    if pattern_file is None:
-        pattern_file = tempfile.NamedTemporaryFile('w')
+    if patterns_file is None:
+        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
+        operation_name = 'Writing'
     else:
-        pattern_file.seek(0)
+        patterns_file.write('\n')
+        operation_name = 'Appending'
 
-    pattern_file.write(
-        '\n'.join(tuple(patterns or ()) + tuple(f'R {source}' for source in (sources or [])))
+    patterns_output = '\n'.join(
+        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
+        for pattern in patterns
     )
-    pattern_file.flush()
+    logger.debug(f'{operation_name} patterns to {patterns_file.name}:\n{patterns_output}')
 
-    return pattern_file
+    patterns_file.write(patterns_output)
+    patterns_file.flush()
+
+    return patterns_file
 
 
-def ensure_files_readable(*filename_lists):
+def make_exclude_flags(config):
     '''
-    Given a sequence of filename sequences, ensure that each filename is openable. This prevents
-    unreadable files from being passed to Borg, which in certain situations only warns instead of
-    erroring.
+    Given a configuration dict with various exclude options, return the corresponding Borg flags as
+    a tuple.
     '''
-    for file_object in itertools.chain.from_iterable(
-        filename_list for filename_list in filename_lists if filename_list
-    ):
-        open(file_object).close()
-
-
-def make_pattern_flags(location_config, pattern_filename=None):
-    '''
-    Given a location config dict with a potential patterns_from option, and a filename containing
-    any additional patterns, return the corresponding Borg flags for those files as a tuple.
-    '''
-    pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
-        (pattern_filename,) if pattern_filename else ()
-    )
-
-    return tuple(
-        itertools.chain.from_iterable(
-            ('--patterns-from', pattern_filename) for pattern_filename in pattern_filenames
-        )
-    )
-
-
-def make_exclude_flags(location_config, exclude_filename=None):
-    '''
-    Given a location config dict with various exclude options, and a filename containing any exclude
-    patterns, return the corresponding Borg flags as a tuple.
-    '''
-    exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
-        (exclude_filename,) if exclude_filename else ()
-    )
-    exclude_from_flags = tuple(
-        itertools.chain.from_iterable(
-            ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
-        )
-    )
-    caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
+    caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
     if_present_flags = tuple(
         itertools.chain.from_iterable(
             ('--exclude-if-present', if_present)
-            for if_present in location_config.get('exclude_if_present', ())
+            for if_present in config.get('exclude_if_present', ())
         )
     )
-    keep_exclude_tags_flags = (
-        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
-    )
-    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
+    keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
+    exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
 
-    return (
-        exclude_from_flags
-        + caches_flag
-        + if_present_flags
-        + keep_exclude_tags_flags
-        + exclude_nodump_flags
-    )
+    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
 
 
 def make_list_filter_flags(local_borg_version, dry_run):
@@ -217,47 +90,14 @@ def make_list_filter_flags(local_borg_version, dry_run):
         return f'{base_flags}-'
 
 
-DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
-
-
-def collect_borgmatic_source_directories(borgmatic_source_directory):
-    '''
-    Return a list of borgmatic-specific source directories used for state like database backups.
-    '''
-    if not borgmatic_source_directory:
-        borgmatic_source_directory = state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
-
-    return (
-        [borgmatic_source_directory]
-        if os.path.exists(os.path.expanduser(borgmatic_source_directory))
-        else []
-    )
-
-
-ROOT_PATTERN_PREFIX = 'R '
-
-
-def pattern_root_directories(patterns=None):
-    '''
-    Given a sequence of patterns, parse out and return just the root directories.
-    '''
-    if not patterns:
-        return []
-
-    return [
-        pattern.split(ROOT_PATTERN_PREFIX, maxsplit=1)[1]
-        for pattern in patterns
-        if pattern.startswith(ROOT_PATTERN_PREFIX)
-    ]
-
-
-def special_file(path):
+def special_file(path, working_directory=None):
     '''
     Return whether the given path is a special file (character device, block device, or named pipe
-    / FIFO).
+    / FIFO). If a working directory is given, take it into account when making the full path to
+    check.
     '''
     try:
-        mode = os.stat(path).st_mode
+        mode = os.stat(os.path.join(working_directory or '', path)).st_mode
     except (FileNotFoundError, OSError):
         return False
 
@@ -277,129 +117,147 @@ def any_parent_directories(path, candidate_parents):
 
 
 def collect_special_file_paths(
-    create_command, local_path, working_directory, borg_environment, skip_directories
+    dry_run,
+    create_command,
+    config,
+    local_path,
+    working_directory,
+    borgmatic_runtime_directory,
 ):
     '''
-    Given a Borg create command as a tuple, a local Borg path, a working directory, and a dict of
-    environment variables to pass to Borg, and a sequence of parent directories to skip, collect the
-    paths for any special files (character devices, block devices, and named pipes / FIFOs) that
-    Borg would encounter during a create. These are all paths that could cause Borg to hang if its
-    --read-special flag is used.
+    Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
+    a working directory, and the borgmatic runtime directory, collect the paths for any special
+    files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
+    during a create. These are all paths that could cause Borg to hang if its --read-special flag is
+    used.
+
+    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
+    its own special files there for database dumps and we don't want those omitted.
+
+    Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
+    plans to backup, that means the user must have excluded the runtime directory (e.g. via
+    "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
+    consume any database dumps and therefore borgmatic will hang when it tries to do so.
     '''
+    # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
+    # files including any named pipe we've created. And omit "--filter" because that can break the
+    # paths output parsing below such that path lines no longer start with th expected "- ".
     paths_output = execute_command_and_capture_output(
-        create_command + ('--dry-run', '--list'),
+        flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
+        + ('--dry-run', '--list'),
         capture_stderr=True,
         working_directory=working_directory,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
     )
 
+    # These are all the individual files that Borg is planning to backup as determined by the Borg
+    # create dry run above.
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.split('\n')
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
 
+    # These are the subset of those files that contain the borgmatic runtime directory.
+    paths_containing_runtime_directory = {}
+
+    if os.path.exists(borgmatic_runtime_directory):
+        paths_containing_runtime_directory = {
+            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
+        }
+
+        # If no paths to backup contain the runtime directory, it must've been excluded.
+        if not paths_containing_runtime_directory and not dry_run:
+            raise ValueError(
+                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
+            )
+
     return tuple(
         path
         for path in paths
-        if special_file(path) and not any_parent_directories(path, skip_directories)
+        if special_file(path, working_directory)
+        if path not in paths_containing_runtime_directory
     )
 
 
-def check_all_source_directories_exist(source_directories):
+def check_all_root_patterns_exist(patterns):
     '''
-    Given a sequence of source directories, check that they all exist. If any do not, raise an
-    exception.
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
+    paths exist. If any don't, raise an exception.
     '''
-    missing_directories = [
-        source_directory
-        for source_directory in source_directories
-        if not all([os.path.exists(directory) for directory in expand_directory(source_directory)])
+    missing_paths = [
+        pattern.path
+        for pattern in patterns
+        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+        if not os.path.exists(pattern.path)
     ]
-    if missing_directories:
-        raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
+
+    if missing_paths:
+        raise ValueError(
+            f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}"
+        )
 
 
-def create_archive(
+MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
+
+
+def make_base_create_command(
     dry_run,
     repository_path,
-    location_config,
-    storage_config,
+    config,
+    patterns,
     local_borg_version,
     global_arguments,
+    borgmatic_runtime_directory,
     local_path='borg',
     remote_path=None,
     progress=False,
-    stats=False,
     json=False,
     list_files=False,
     stream_processes=None,
 ):
     '''
-    Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
-    storage config dict, create a Borg archive and return Borg's JSON output (if any).
-
-    If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
-    create command while also triggering the given processes to produce output.
+    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
+    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version,
+    global arguments as an argparse.Namespace instance, and a sequence of borgmatic source
+    directories, return a tuple of (base Borg create command flags, Borg create command positional
+    arguments, open pattern file handle).
     '''
-    borgmatic.logger.add_custom_log_levels()
-    borgmatic_source_directories = expand_directories(
-        collect_borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
-    )
-    if location_config.get('source_directories_must_exist', False):
-        check_all_source_directories_exist(location_config.get('source_directories'))
-    sources = deduplicate_directories(
-        map_directories_to_devices(
-            expand_directories(
-                tuple(location_config.get('source_directories', ())) + borgmatic_source_directories
-            )
-        ),
-        additional_directory_devices=map_directories_to_devices(
-            expand_directories(pattern_root_directories(location_config.get('patterns')))
-        ),
-    )
+    if config.get('source_directories_must_exist', False):
+        check_all_root_patterns_exist(patterns)
 
-    ensure_files_readable(location_config.get('patterns_from'), location_config.get('exclude_from'))
-
-    try:
-        working_directory = os.path.expanduser(location_config.get('working_directory'))
-    except TypeError:
-        working_directory = None
-
-    pattern_file = (
-        write_pattern_file(location_config.get('patterns'), sources)
-        if location_config.get('patterns') or location_config.get('patterns_from')
-        else None
-    )
-    exclude_file = write_pattern_file(
-        expand_home_directories(location_config.get('exclude_patterns'))
-    )
-    checkpoint_interval = storage_config.get('checkpoint_interval', None)
-    checkpoint_volume = storage_config.get('checkpoint_volume', None)
-    chunker_params = storage_config.get('chunker_params', None)
-    compression = storage_config.get('compression', None)
-    upload_rate_limit = storage_config.get('upload_rate_limit', None)
-    umask = storage_config.get('umask', None)
-    lock_wait = storage_config.get('lock_wait', None)
+    patterns_file = write_patterns_file(patterns, borgmatic_runtime_directory)
+    checkpoint_interval = config.get('checkpoint_interval', None)
+    checkpoint_volume = config.get('checkpoint_volume', None)
+    chunker_params = config.get('chunker_params', None)
+    compression = config.get('compression', None)
+    upload_rate_limit = config.get('upload_rate_limit', None)
+    upload_buffer_size = config.get('upload_buffer_size', None)
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
     list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
-    files_cache = location_config.get('files_cache')
-    archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
-    extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
+    files_cache = config.get('files_cache')
+    archive_name_format = config.get(
+        'archive_name_format', flags.get_default_archive_name_format(local_borg_version)
+    )
+    extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
 
     if feature.available(feature.Feature.ATIME, local_borg_version):
-        atime_flags = ('--atime',) if location_config.get('atime') is True else ()
+        atime_flags = ('--atime',) if config.get('atime') is True else ()
     else:
-        atime_flags = ('--noatime',) if location_config.get('atime') is False else ()
+        atime_flags = ('--noatime',) if config.get('atime') is False else ()
 
     if feature.available(feature.Feature.NOFLAGS, local_borg_version):
-        noflags_flags = ('--noflags',) if location_config.get('flags') is False else ()
+        noflags_flags = ('--noflags',) if config.get('flags') is False else ()
     else:
-        noflags_flags = ('--nobsdflags',) if location_config.get('flags') is False else ()
+        noflags_flags = ('--nobsdflags',) if config.get('flags') is False else ()
 
     if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
-        numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_ids') else ()
+        numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
     else:
-        numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else ()
+        numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
 
     if feature.available(feature.Feature.UPLOAD_RATELIMIT, local_borg_version):
         upload_ratelimit_flags = (
@@ -410,31 +268,23 @@ def create_archive(
             ('--remote-ratelimit', str(upload_rate_limit)) if upload_rate_limit else ()
         )
 
-    if stream_processes and location_config.get('read_special') is False:
-        logger.warning(
-            f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
-        )
-
-    create_command = (
+    create_flags = (
         tuple(local_path.split(' '))
         + ('create',)
-        + make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
-        + make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
+        + (('--patterns-from', patterns_file.name) if patterns_file else ())
+        + make_exclude_flags(config)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + upload_ratelimit_flags
-        + (
-            ('--one-file-system',)
-            if location_config.get('one_file_system') or stream_processes
-            else ()
-        )
+        + (('--upload-buffer', str(upload_buffer_size)) if upload_buffer_size else ())
+        + (('--one-file-system',) if config.get('one_file_system') else ())
         + numeric_ids_flags
         + atime_flags
-        + (('--noctime',) if location_config.get('ctime') is False else ())
-        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
-        + (('--read-special',) if location_config.get('read_special') or stream_processes else ())
+        + (('--noctime',) if config.get('ctime') is False else ())
+        + (('--nobirthtime',) if config.get('birthtime') is False else ())
+        + (('--read-special',) if config.get('read_special') or stream_processes else ())
         + noflags_flags
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
@@ -448,10 +298,101 @@ def create_archive(
         )
         + (('--dry-run',) if dry_run else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_archive_flags(
-            repository_path, archive_name_format, local_borg_version
+    )
+
+    create_positional_arguments = flags.make_repository_archive_flags(
+        repository_path, archive_name_format, local_borg_version
+    )
+
+    # If database hooks are enabled (as indicated by streaming processes), exclude files that might
+    # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
+    if stream_processes and not config.get('read_special'):
+        logger.warning(
+            'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
-        + (sources if not pattern_file else ())
+        working_directory = borgmatic.config.paths.get_working_directory(config)
+
+        logger.debug('Collecting special file paths')
+        special_file_paths = collect_special_file_paths(
+            dry_run,
+            create_flags + create_positional_arguments,
+            config,
+            local_path,
+            working_directory,
+            borgmatic_runtime_directory=borgmatic_runtime_directory,
+        )
+
+        if special_file_paths:
+            truncated_special_file_paths = textwrap.shorten(
+                ', '.join(special_file_paths),
+                width=MAX_SPECIAL_FILE_PATHS_LENGTH,
+                placeholder=' ...',
+            )
+            logger.warning(
+                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
+            )
+            patterns_file = write_patterns_file(
+                tuple(
+                    borgmatic.borg.pattern.Pattern(
+                        special_file_path,
+                        borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
+                        borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                        source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
+                    )
+                    for special_file_path in special_file_paths
+                ),
+                borgmatic_runtime_directory,
+                patterns_file=patterns_file,
+            )
+
+            if '--patterns-from' not in create_flags:
+                create_flags += ('--patterns-from', patterns_file.name)
+
+    return (create_flags, create_positional_arguments, patterns_file)
+
+
+def create_archive(
+    dry_run,
+    repository_path,
+    config,
+    patterns,
+    local_borg_version,
+    global_arguments,
+    borgmatic_runtime_directory,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+    stats=False,
+    json=False,
+    list_files=False,
+    stream_processes=None,
+):
+    '''
+    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
+    sequence of loaded configuration paths, the local Borg version, and global arguments as an
+    argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
+
+    If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
+    create command while also triggering the given processes to produce output.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    (create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
+        dry_run,
+        repository_path,
+        config,
+        patterns,
+        local_borg_version,
+        global_arguments,
+        borgmatic_runtime_directory,
+        local_path,
+        remote_path,
+        progress,
+        json,
+        list_files,
+        stream_processes,
     )
 
     if json:
@@ -465,62 +406,41 @@ def create_archive(
     # the terminal directly.
     output_file = DO_NOT_CAPTURE if progress else None
 
-    borg_environment = environment.make_environment(storage_config)
-
-    # If database hooks are enabled (as indicated by streaming processes), exclude files that might
-    # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
-    if stream_processes and not location_config.get('read_special'):
-        logger.debug(f'{repository_path}: Collecting special file paths')
-        special_file_paths = collect_special_file_paths(
-            create_command,
-            local_path,
-            working_directory,
-            borg_environment,
-            skip_directories=borgmatic_source_directories,
-        )
-
-        if special_file_paths:
-            logger.warning(
-                f'{repository_path}: Excluding special files to prevent Borg from hanging: {", ".join(special_file_paths)}'
-            )
-            exclude_file = write_pattern_file(
-                expand_home_directories(
-                    tuple(location_config.get('exclude_patterns') or ()) + special_file_paths
-                ),
-                pattern_file=exclude_file,
-            )
-            create_command += make_exclude_flags(location_config, exclude_file.name)
-
-    create_command += (
+    create_flags += (
         (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
         + (('--stats',) if stats and not json and not dry_run else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
         + (('--progress',) if progress else ())
         + (('--json',) if json else ())
     )
+    borg_exit_codes = config.get('borg_exit_codes')
 
     if stream_processes:
         return execute_command_with_processes(
-            create_command,
+            create_flags + create_positional_arguments,
             stream_processes,
             output_log_level,
             output_file,
-            borg_local_path=local_path,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
         )
     elif output_log_level is None:
         return execute_command_and_capture_output(
-            create_command,
+            create_flags + create_positional_arguments,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
         )
     else:
         execute_command(
-            create_command,
+            create_flags + create_positional_arguments,
             output_log_level,
             output_file,
-            borg_local_path=local_path,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/delete.py

@@ -0,0 +1,135 @@
+import argparse
+import logging
+
+import borgmatic.borg.environment
+import borgmatic.borg.feature
+import borgmatic.borg.flags
+import borgmatic.borg.repo_delete
+import borgmatic.config.paths
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def make_delete_command(
+    repository,
+    config,
+    local_borg_version,
+    delete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the delete action as an argparse.Namespace, and global arguments, return a command
+    as a tuple to delete archives from the repository.
+    '''
+    return (
+        (local_path, 'delete')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
+        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
+        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
+        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+        + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives)
+        + (
+            (('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
+            if delete_arguments.force
+            else ()
+        )
+        # Ignore match_archives and archive_name_format options from configuration, so the user has
+        # to be explicit on the command-line about the archives they want to delete.
+        + borgmatic.borg.flags.make_match_archives_flags(
+            delete_arguments.match_archives or delete_arguments.archive,
+            archive_name_format=None,
+            local_borg_version=local_borg_version,
+            default_archive_name_format='*',
+        )
+        + borgmatic.borg.flags.make_flags_from_arguments(
+            delete_arguments,
+            excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'),
+        )
+        + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
+    )
+
+
+ARCHIVE_RELATED_ARGUMENT_NAMES = (
+    'archive',
+    'match_archives',
+    'first',
+    'last',
+    'oldest',
+    'newest',
+    'older',
+    'newer',
+)
+
+
+def delete_archives(
+    repository,
+    config,
+    local_borg_version,
+    delete_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the delete action as an argparse.Namespace, global arguments as an
+    argparse.Namespace, and local and remote Borg paths, delete the selected archives from the
+    repository. If no archives are selected, then delete the entire repository.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    if not any(
+        getattr(delete_arguments, argument_name, None)
+        for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
+    ):
+        if borgmatic.borg.feature.available(
+            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+        ):
+            logger.warning(
+                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.'
+            )
+
+        repo_delete_arguments = argparse.Namespace(
+            repository=repository['path'],
+            list_archives=delete_arguments.list_archives,
+            force=delete_arguments.force,
+            cache_only=delete_arguments.cache_only,
+            keep_security_info=delete_arguments.keep_security_info,
+        )
+        borgmatic.borg.repo_delete.delete_repository(
+            repository,
+            config,
+            local_borg_version,
+            repo_delete_arguments,
+            global_arguments,
+            local_path,
+            remote_path,
+        )
+
+        return
+
+    command = make_delete_command(
+        repository,
+        config,
+        local_borg_version,
+        delete_arguments,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.execute.execute_command(
+        command,
+        output_log_level=logging.ANSWER,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/environment.py

@@ -1,3 +1,8 @@
+import os
+
+import borgmatic.borg.passcommand
+import borgmatic.hooks.credential.parse
+
 OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
     'borg_config_directory': 'BORG_CONFIG_DIR',
@@ -5,36 +10,93 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
-    'encryption_passcommand': 'BORG_PASSCOMMAND',
-    'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
 }
 
-DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
+DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = {
     'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
     'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
 }
 
+DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
+    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
+}
 
-def make_environment(storage_config):
+
+def make_environment(config):
     '''
-    Given a borgmatic storage configuration dict, return its options converted to a Borg environment
-    variable dict.
+    Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it
+    with a copy of the current environment variables, and return the result.
+
+    Do not reuse this environment across multiple Borg invocations, because it can include
+    references to resources like anonymous pipes for passphrases—which can only be consumed once.
+
+    Here's how native Borg precedence works for a few of the environment variables:
+
+      1. BORG_PASSPHRASE, if set, is used first.
+      2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set.
+      3. BORG_PASSPHRASE_FD is used only if neither of the above are set.
+
+    In borgmatic, we want to simulate this precedence order, but there are some additional
+    complications. First, values can come from either configuration or from environment variables
+    set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a
+    passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we
+    want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's
+    more secure than using an environment variable (BORG_PASSPHRASE).
     '''
-    environment = {}
+    environment = dict(os.environ)
 
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
-        value = storage_config.get(option_name)
+        value = config.get(option_name)
 
-        if value:
+        if value is not None:
             environment[environment_variable_name] = str(value)
 
+    if 'encryption_passphrase' in config:
+        environment.pop('BORG_PASSPHRASE', None)
+        environment.pop('BORG_PASSCOMMAND', None)
+
+    if 'encryption_passcommand' in config:
+        environment.pop('BORG_PASSCOMMAND', None)
+
+    passphrase = borgmatic.hooks.credential.parse.resolve_credential(
+        config.get('encryption_passphrase'), config
+    )
+
+    if passphrase is None:
+        passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
+
+    # If there's a passphrase (from configuration, from a configured credential, or from a
+    # configured passcommand), send it to Borg via an anonymous pipe.
+    if passphrase is not None:
+        read_file_descriptor, write_file_descriptor = os.pipe()
+        os.write(write_file_descriptor, passphrase.encode('utf-8'))
+        os.close(write_file_descriptor)
+
+        # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg
+        # child process to inherit the file descriptor.
+        os.set_inheritable(read_file_descriptor, True)
+        environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor)
+
     for (
         option_name,
         environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
-        value = storage_config.get(option_name, False)
-        environment[environment_variable_name] = 'yes' if value else 'no'
+    ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
+        if os.environ.get(environment_variable_name) is None:
+            value = config.get(option_name)
+            environment[environment_variable_name] = 'yes' if value else 'no'
+
+    for (
+        option_name,
+        environment_variable_name,
+    ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items():
+        value = config.get(option_name)
+        if value is not None:
+            environment[environment_variable_name] = 'YES' if value else 'NO'
+
+    # On Borg 1.4.0a1+, take advantage of more specific exit codes. No effect on
+    # older versions of Borg.
+    environment['BORG_EXIT_CODES'] = 'modern'
 
     return environment

borgmatic/borg/export_key.py

@@ -0,0 +1,74 @@
+import logging
+import os
+
+import borgmatic.config.paths
+import borgmatic.logger
+from borgmatic.borg import environment, flags
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def export_key(
+    repository_path,
+    config,
+    local_borg_version,
+    export_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, export
+    arguments, and optional local and remote Borg paths, export the repository key to the
+    destination path indicated in the export arguments.
+
+    If the destination path is empty or "-", then print the key to stdout instead of to a file.
+
+    Raise FileExistsError if a path is given but it already exists on disk.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    if export_arguments.path and export_arguments.path != '-':
+        if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
+            raise FileExistsError(
+                f'Destination path {export_arguments.path} already exists. Aborting.'
+            )
+
+        output_file = None
+    else:
+        output_file = DO_NOT_CAPTURE
+
+    full_command = (
+        (local_path, 'key', 'export')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + flags.make_flags('paper', export_arguments.paper)
+        + flags.make_flags('qr-html', export_arguments.qr_html)
+        + flags.make_repository_flags(
+            repository_path,
+            local_borg_version,
+        )
+        + ((export_arguments.path,) if output_file is None else ())
+    )
+
+    if global_arguments.dry_run:
+        logger.info('Skipping key export (dry run)')
+        return
+
+    execute_command(
+        full_command,
+        output_file=output_file,
+        output_log_level=logging.ANSWER,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/export_tar.py

@@ -1,5 +1,6 @@
 import logging
 
+import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@@ -13,7 +14,7 @@ def export_tar_archive(
     archive,
     paths,
     destination_path,
-    storage_config,
+    config,
     local_borg_version,
     global_arguments,
     local_path='borg',
@@ -24,16 +25,16 @@ def export_tar_archive(
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    export from the archive, a destination path to export to, a storage configuration dict, the
-    local Borg version, optional local and remote Borg paths, an optional filter program, whether to
-    include per-file details, and an optional number of path components to strip, export the archive
-    into the given destination path as a tar-formatted file.
+    export from the archive, a destination path to export to, a configuration dict, the local Borg
+    version, optional local and remote Borg paths, an optional filter program, whether to include
+    per-file details, and an optional number of path components to strip, export the archive into
+    the given destination path as a tar-formatted file.
 
     If the destination path is "-", then stream the output to stdout instead of to a file.
     '''
     borgmatic.logger.add_custom_log_levels()
-    umask = storage_config.get('umask', None)
-    lock_wait = storage_config.get('lock_wait', None)
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
 
     full_command = (
         (local_path, 'export-tar')
@@ -62,13 +63,15 @@ def export_tar_archive(
         output_log_level = logging.INFO
 
     if dry_run:
-        logging.info(f'{repository_path}: Skipping export to tar file (dry run)')
+        logging.info('Skipping export to tar file (dry run)')
         return
 
     execute_command(
         full_command,
         output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
         output_log_level=output_log_level,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
+        borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/extract.py

@@ -2,15 +2,16 @@ import logging
 import os
 import subprocess
 
+import borgmatic.config.paths
 import borgmatic.config.validate
-from borgmatic.borg import environment, feature, flags, rlist
+from borgmatic.borg import environment, feature, flags, repo_list
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
 
 def extract_last_archive_dry_run(
-    storage_config,
+    config,
     local_borg_version,
     global_arguments,
     repository_path,
@@ -29,10 +30,10 @@ def extract_last_archive_dry_run(
         verbosity_flags = ('--info',)
 
     try:
-        last_archive_name = rlist.resolve_archive_name(
+        last_archive_name = repo_list.resolve_archive_name(
             repository_path,
             'latest',
-            storage_config,
+            config,
             local_borg_version,
             global_arguments,
             local_path,
@@ -43,7 +44,6 @@ def extract_last_archive_dry_run(
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
-    borg_environment = environment.make_environment(storage_config)
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
         + (('--remote-path', remote_path) if remote_path else ())
@@ -57,7 +57,11 @@ def extract_last_archive_dry_run(
     )
 
     execute_command(
-        full_extract_command, working_directory=None, extra_environment=borg_environment
+        full_extract_command,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
     )
 
 
@@ -66,8 +70,7 @@ def extract_archive(
     repository,
     archive,
     paths,
-    location_config,
-    storage_config,
+    config,
     local_borg_version,
     global_arguments,
     local_path='borg',
@@ -80,29 +83,39 @@ def extract_archive(
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
     restore from the archive, the local Borg version string, an argparse.Namespace of global
-    arguments, location/storage configuration dicts, optional local and remote Borg paths, and an
-    optional destination path to extract to, extract the archive into the current directory.
+    arguments, a configuration dict, optional local and remote Borg paths, and an optional
+    destination path to extract to, extract the archive into the current directory.
 
     If extract to stdout is True, then start the extraction streaming to stdout, and return that
     extract process as an instance of subprocess.Popen.
     '''
-    umask = storage_config.get('umask', None)
-    lock_wait = storage_config.get('lock_wait', None)
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
 
     if progress and extract_to_stdout:
         raise ValueError('progress and extract_to_stdout cannot both be set')
 
     if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
-        numeric_ids_flags = ('--numeric-ids',) if location_config.get('numeric_ids') else ()
+        numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
     else:
-        numeric_ids_flags = ('--numeric-owner',) if location_config.get('numeric_ids') else ()
+        numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
 
     if strip_components == 'all':
         if not paths:
             raise ValueError('The --strip-components flag with "all" requires at least one --path')
 
-        # Calculate the maximum number of leading path components of the given paths.
-        strip_components = max(0, *(len(path.split(os.path.sep)) - 1 for path in paths))
+        # Calculate the maximum number of leading path components of the given paths. "if piece"
+        # ignores empty path components, e.g. those resulting from a leading slash. And the "- 1"
+        # is so this doesn't count the final path component, e.g. the filename itself.
+        strip_components = max(
+            0,
+            *(
+                len(tuple(piece for piece in path.split(os.path.sep) if piece)) - 1
+                for path in paths
+            ),
+        )
+
+    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     full_command = (
         (local_path, 'extract')
@@ -118,16 +131,22 @@ def extract_archive(
         + (('--progress',) if progress else ())
         + (('--stdout',) if extract_to_stdout else ())
         + flags.make_repository_archive_flags(
-            # Make the repository path absolute so the working directory changes below don't
-            # prevent Borg from finding the repo.
-            borgmatic.config.validate.normalize_repository_path(repository),
+            # Make the repository path absolute so the destination directory used below via changing
+            # the working directory doesn't prevent Borg from finding the repo. But also apply the
+            # user's configured working directory (if any) to the repo path.
+            borgmatic.config.validate.normalize_repository_path(
+                os.path.join(working_directory or '', repository)
+            ),
             archive,
             local_borg_version,
         )
         + (tuple(paths) if paths else ())
     )
 
-    borg_environment = environment.make_environment(storage_config)
+    borg_exit_codes = config.get('borg_exit_codes')
+    full_destination_path = (
+        os.path.join(working_directory or '', destination_path) if destination_path else None
+    )
 
     # The progress output isn't compatible with captured and logged output, as progress messes with
     # the terminal directly.
@@ -135,8 +154,10 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            working_directory=destination_path,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
+            working_directory=full_destination_path,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
         )
         return None
 
@@ -144,13 +165,19 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=subprocess.PIPE,
-            working_directory=destination_path,
             run_to_completion=False,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
+            working_directory=full_destination_path,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
         )
 
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
     execute_command(
-        full_command, working_directory=destination_path, extra_environment=borg_environment
+        full_command,
+        environment=environment.make_environment(config),
+        working_directory=full_destination_path,
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
     )

borgmatic/borg/feature.py

@@ -10,11 +10,13 @@ class Feature(Enum):
     NUMERIC_IDS = 4
     UPLOAD_RATELIMIT = 5
     SEPARATE_REPOSITORY_ARCHIVE = 6
-    RCREATE = 7
-    RLIST = 8
-    RINFO = 9
-    MATCH_ARCHIVES = 10
-    EXCLUDED_FILES_MINUS = 11
+    REPO_CREATE = 7
+    REPO_LIST = 8
+    REPO_INFO = 9
+    REPO_DELETE = 10
+    MATCH_ARCHIVES = 11
+    EXCLUDED_FILES_MINUS = 12
+    ARCHIVE_SERIES = 13
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
@@ -24,11 +26,13 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
     Feature.NUMERIC_IDS: parse('1.2.0b3'),  # borg create/extract/mount --numeric-ids
     Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'),  # borg create --upload-ratelimit
     Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'),  # --repo with separate archive
-    Feature.RCREATE: parse('2.0.0a2'),  # borg rcreate
-    Feature.RLIST: parse('2.0.0a2'),  # borg rlist
-    Feature.RINFO: parse('2.0.0a2'),  # borg rinfo
+    Feature.REPO_CREATE: parse('2.0.0a2'),  # borg repo-create
+    Feature.REPO_LIST: parse('2.0.0a2'),  # borg repo-list
+    Feature.REPO_INFO: parse('2.0.0a2'),  # borg repo-info
+    Feature.REPO_DELETE: parse('2.0.0a2'),  # borg repo-delete
     Feature.MATCH_ARCHIVES: parse('2.0.0b3'),  # borg --match-archives
     Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
+    Feature.ARCHIVE_SERIES: parse('2.0.0b11'),  # identically named archives form a series
 }
 
 

borgmatic/borg/flags.py

@@ -1,8 +1,12 @@
 import itertools
+import json
+import logging
 import re
 
 from borgmatic.borg import feature
 
+logger = logging.getLogger(__name__)
+
 
 def make_flags(name, value):
     '''
@@ -46,6 +50,9 @@ def make_repository_flags(repository_path, local_borg_version):
     ) + (repository_path,)
 
 
+ARCHIVE_HASH_PATTERN = re.compile('[0-9a-fA-F]{8,}$')
+
+
 def make_repository_archive_flags(repository_path, archive, local_borg_version):
     '''
     Given the path of a Borg repository, an archive name or pattern, and the local Borg version,
@@ -53,31 +60,140 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
     and archive.
     '''
     return (
-        ('--repo', repository_path, archive)
+        (
+            '--repo',
+            repository_path,
+            (
+                f'aid:{archive}'
+                if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
+                and ARCHIVE_HASH_PATTERN.match(archive)
+                and not archive.startswith('aid:')
+                else archive
+            ),
+        )
         if feature.available(feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version)
         else (f'{repository_path}::{archive}',)
     )
 
 
-def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
+DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
+DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'  # noqa: FS003
+
+
+def get_default_archive_name_format(local_borg_version):
+    '''
+    Given the local Borg version, return the corresponding default archive name format.
+    '''
+    if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version):
+        return DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES
+
+    return DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES
+
+
+def make_match_archives_flags(
+    match_archives,
+    archive_name_format,
+    local_borg_version,
+    default_archive_name_format=None,
+):
     '''
     Return match archives flags based on the given match archives value, if any. If it isn't set,
-    return match archives flags to match archives created with the given archive name format, if
-    any. This is done by replacing certain archive name format placeholders for ephemeral data (like
-    "{now}") with globs.
+    return match archives flags to match archives created with the given (or default) archive name
+    format. This is done by replacing certain archive name format placeholders for ephemeral data
+    (like "{now}") with globs.
     '''
     if match_archives:
+        if match_archives in {'*', 're:.*', 'sh:*'}:
+            return ()
+
         if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
+            if (
+                feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
+                and ARCHIVE_HASH_PATTERN.match(match_archives)
+                and not match_archives.startswith('aid:')
+            ):
+                return ('--match-archives', f'aid:{match_archives}')
+
             return ('--match-archives', match_archives)
         else:
             return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
 
-    if not archive_name_format:
-        return ()
+    derived_match_archives = re.sub(
+        r'\{(now|utcnow|pid)([:%\w\.-]*)\}',
+        '*',
+        archive_name_format
+        or default_archive_name_format
+        or get_default_archive_name_format(local_borg_version),
+    )
 
-    derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
+    if derived_match_archives == '*':
+        return ()
 
     if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
         return ('--match-archives', f'sh:{derived_match_archives}')
     else:
         return ('--glob-archives', f'{derived_match_archives}')
+
+
+def warn_for_aggressive_archive_flags(json_command, json_output):
+    '''
+    Given a JSON archives command and the resulting JSON string output from running it, parse the
+    JSON and warn if the command used an archive flag but the output indicates zero archives were
+    found.
+    '''
+    archive_flags_used = {'--glob-archives', '--match-archives'}.intersection(set(json_command))
+
+    if not archive_flags_used:
+        return
+
+    try:
+        if len(json.loads(json_output)['archives']) == 0:
+            logger.warning('An archive filter was applied, but no matching archives were found.')
+            logger.warning(
+                'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.'
+            )
+    except json.JSONDecodeError as error:
+        logger.debug(f'Cannot parse JSON output from archive command: {error}')
+    except (TypeError, KeyError):
+        logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
+
+
+def omit_flag(arguments, flag):
+    '''
+    Given a sequence of Borg command-line arguments, return them with the given (valueless) flag
+    omitted. For instance, if the flag is "--flag" and arguments is:
+
+        ('borg', 'create', '--flag', '--other-flag')
+
+    ... then return:
+
+        ('borg', 'create', '--other-flag')
+    '''
+    return tuple(argument for argument in arguments if argument != flag)
+
+
+def omit_flag_and_value(arguments, flag):
+    '''
+    Given a sequence of Borg command-line arguments, return them with the given flag and its
+    corresponding value omitted. For instance, if the flag is "--flag" and arguments is:
+
+        ('borg', 'create', '--flag', 'value', '--other-flag')
+
+    ... or:
+
+        ('borg', 'create', '--flag=value', '--other-flag')
+
+    ... then return:
+
+        ('borg', 'create', '--other-flag')
+    '''
+    # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two',
+    # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')).
+    # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and
+    # its value.
+    return tuple(
+        argument
+        for (previous_argument, argument) in zip((None,) + arguments, arguments)
+        if flag not in (previous_argument, argument)
+        if not argument.startswith(f'{flag}=')
+    )

borgmatic/borg/info.py

@@ -1,5 +1,7 @@
+import argparse
 import logging
 
+import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command, execute_command_and_capture_output
@@ -7,24 +9,21 @@ from borgmatic.execute import execute_command, execute_command_and_capture_outpu
 logger = logging.getLogger(__name__)
 
 
-def display_archives_info(
+def make_info_command(
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
     info_arguments,
     global_arguments,
-    local_path='borg',
-    remote_path=None,
+    local_path,
+    remote_path,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, global
-    arguments as an argparse.Namespace, and the arguments to the info action, display summary
-    information for Borg archives in the repository or return JSON summary information.
+    Given a local or remote repository path, a configuration dict, the local Borg version, the
+    arguments to the info action as an argparse.Namespace, and global arguments, return a command
+    as a tuple to display summary information for archives in the repository.
     '''
-    borgmatic.logger.add_custom_log_levels()
-    lock_wait = storage_config.get('lock_wait', None)
-
-    full_command = (
+    return (
         (local_path, 'info')
         + (
             ('--info',)
@@ -37,8 +36,9 @@ def display_archives_info(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', lock_wait)
+        + flags.make_flags('lock-wait', config.get('lock_wait'))
         + (
             (
                 flags.make_flags('match-archives', f'sh:{info_arguments.prefix}*')
@@ -50,8 +50,8 @@ def display_archives_info(
                 flags.make_match_archives_flags(
                     info_arguments.match_archives
                     or info_arguments.archive
-                    or storage_config.get('match_archives'),
-                    storage_config.get('archive_name_format'),
+                    or config.get('match_archives'),
+                    config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
@@ -62,15 +62,62 @@ def display_archives_info(
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
+
+def display_archives_info(
+    repository_path,
+    config,
+    local_borg_version,
+    info_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, the
+    arguments to the info action as an argparse.Namespace, and global arguments, display summary
+    information for Borg archives in the repository or return JSON summary information.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    main_command = make_info_command(
+        repository_path,
+        config,
+        local_borg_version,
+        info_arguments,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    json_command = make_info_command(
+        repository_path,
+        config,
+        local_borg_version,
+        argparse.Namespace(**dict(info_arguments.__dict__, json=True)),
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    borg_exit_codes = config.get('borg_exit_codes')
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    json_info = execute_command_and_capture_output(
+        json_command,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )
+
     if info_arguments.json:
-        return execute_command_and_capture_output(
-            full_command,
-            extra_environment=environment.make_environment(storage_config),
-        )
-    else:
-        execute_command(
-            full_command,
-            output_log_level=logging.ANSWER,
-            borg_local_path=local_path,
-            extra_environment=environment.make_environment(storage_config),
-        )
+        return json_info
+
+    flags.warn_for_aggressive_archive_flags(json_command, json_info)
+
+    execute_command(
+        main_command,
+        output_log_level=logging.ANSWER,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )

borgmatic/borg/list.py

@@ -3,26 +3,26 @@ import copy
 import logging
 import re
 
+import borgmatic.config.paths
 import borgmatic.logger
-from borgmatic.borg import environment, feature, flags, rlist
+from borgmatic.borg import environment, feature, flags, repo_list
 from borgmatic.execute import execute_command, execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
 
-ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST = ('prefix', 'match_archives', 'sort_by', 'first', 'last')
+ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST = ('prefix', 'match_archives', 'sort_by', 'first', 'last')
 MAKE_FLAGS_EXCLUDES = (
     'repository',
     'archive',
-    'successful',
     'paths',
     'find_paths',
-) + ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST
+) + ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST
 
 
 def make_list_command(
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
     list_arguments,
     global_arguments,
@@ -30,12 +30,10 @@ def make_list_command(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the arguments to the list
-    action, and local and remote Borg paths, return a command as a tuple to list archives or paths
-    within an archive.
+    Given a local or remote repository path, a configuration dict, the arguments to the list action,
+    and local and remote Borg paths, return a command as a tuple to list archives or paths within an
+    archive.
     '''
-    lock_wait = storage_config.get('lock_wait', None)
-
     return (
         (local_path, 'list')
         + (
@@ -49,8 +47,9 @@ def make_list_command(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', lock_wait)
+        + flags.make_flags('lock-wait', config.get('lock_wait'))
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (
             flags.make_repository_archive_flags(
@@ -80,9 +79,11 @@ def make_find_paths(find_paths):
         return ()
 
     return tuple(
-        find_path
-        if re.compile(r'([-!+RrPp] )|(\w\w:)').match(find_path)
-        else f'sh:**/*{find_path}*/**'
+        (
+            find_path
+            if re.compile(r'([-!+RrPp] )|(\w\w:)').match(find_path)
+            else f'sh:**/*{find_path}*/**'
+        )
         for find_path in find_paths
     )
 
@@ -90,49 +91,52 @@ def make_find_paths(find_paths):
 def capture_archive_listing(
     repository_path,
     archive,
-    storage_config,
+    config,
     local_borg_version,
     global_arguments,
-    list_path=None,
+    list_paths=None,
+    path_format=None,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, an archive name, a storage config dict, the local Borg
-    version, global arguments as an argparse.Namespace, the archive path in which to list files, and
-    local and remote Borg paths, capture the output of listing that archive and return it as a list
-    of file paths.
+    Given a local or remote repository path, an archive name, a configuration
+    dict, the local Borg version, global arguments as an argparse.Namespace,
+    the archive paths (or Borg patterns) in which to list files, the Borg path
+    format to use for the output, and local and remote Borg paths, capture the
+    output of listing that archive and return it as a list of file paths.
     '''
-    borg_environment = environment.make_environment(storage_config)
-
     return tuple(
         execute_command_and_capture_output(
             make_list_command(
                 repository_path,
-                storage_config,
+                config,
                 local_borg_version,
                 argparse.Namespace(
                     repository=repository_path,
                     archive=archive,
-                    paths=[f'sh:{list_path}'],
+                    paths=[path for path in list_paths] if list_paths else None,
                     find_paths=None,
                     json=None,
-                    format='{path}{NL}',  # noqa: FS003
+                    format=path_format or '{path}{NUL}',  # noqa: FS003
                 ),
                 global_arguments,
                 local_path,
                 remote_path,
             ),
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
         )
-        .strip('\n')
-        .split('\n')
+        .strip('\0')
+        .split('\0')
     )
 
 
 def list_archive(
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
     list_arguments,
     global_arguments,
@@ -140,22 +144,22 @@ def list_archive(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, global
-    arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace,
-    and local and remote Borg paths, display the output of listing the files of a Borg archive (or
-    return JSON output). If list_arguments.find_paths are given, list the files by searching across
-    multiple archives. If neither find_paths nor archive name are given, instead list the archives
-    in the given repository.
+    Given a local or remote repository path, a configuration dict, the local Borg version, the
+    arguments to the list action as an argparse.Namespace, global arguments as an
+    argparse.Namespace, and local and remote Borg paths, display the output of listing the files of
+    a Borg archive (or return JSON output). If list_arguments.find_paths are given, list the files
+    by searching across multiple archives. If neither find_paths nor archive name are given, instead
+    list the archives in the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
 
     if not list_arguments.archive and not list_arguments.find_paths:
-        if feature.available(feature.Feature.RLIST, local_borg_version):
+        if feature.available(feature.Feature.REPO_LIST, local_borg_version):
             logger.warning(
-                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the rlist action instead.'
+                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.'
             )
 
-        rlist_arguments = argparse.Namespace(
+        repo_list_arguments = argparse.Namespace(
             repository=repository_path,
             short=list_arguments.short,
             format=list_arguments.format,
@@ -166,18 +170,18 @@ def list_archive(
             first=list_arguments.first,
             last=list_arguments.last,
         )
-        return rlist.list_repository(
+        return repo_list.list_repository(
             repository_path,
-            storage_config,
+            config,
             local_borg_version,
-            rlist_arguments,
+            repo_list_arguments,
             global_arguments,
             local_path,
             remote_path,
         )
 
     if list_arguments.archive:
-        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST:
+        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
             if getattr(list_arguments, name, None):
                 logger.warning(
                     f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag."
@@ -188,12 +192,12 @@ def list_archive(
             'The --json flag on the list action is not supported when using the --archive/--find flags.'
         )
 
-    borg_environment = environment.make_environment(storage_config)
+    borg_exit_codes = config.get('borg_exit_codes')
 
     # If there are any paths to find (and there's not a single archive already selected), start by
     # getting a list of archives to search.
     if list_arguments.find_paths and not list_arguments.archive:
-        rlist_arguments = argparse.Namespace(
+        repo_list_arguments = argparse.Namespace(
             repository=repository_path,
             short=True,
             format=None,
@@ -208,38 +212,41 @@ def list_archive(
         # Ask Borg to list archives. Capture its output for use below.
         archive_lines = tuple(
             execute_command_and_capture_output(
-                rlist.make_rlist_command(
+                repo_list.make_repo_list_command(
                     repository_path,
-                    storage_config,
+                    config,
                     local_borg_version,
-                    rlist_arguments,
+                    repo_list_arguments,
                     global_arguments,
                     local_path,
                     remote_path,
                 ),
-                extra_environment=borg_environment,
+                environment=environment.make_environment(config),
+                working_directory=borgmatic.config.paths.get_working_directory(config),
+                borg_local_path=local_path,
+                borg_exit_codes=borg_exit_codes,
             )
             .strip('\n')
-            .split('\n')
+            .splitlines()
         )
     else:
         archive_lines = (list_arguments.archive,)
 
     # For each archive listed by Borg, run list on the contents of that archive.
     for archive in archive_lines:
-        logger.answer(f'{repository_path}: Listing archive {archive}')
+        logger.answer(f'Listing archive {archive}')
 
         archive_arguments = copy.copy(list_arguments)
         archive_arguments.archive = archive
 
         # This list call is to show the files in a single archive, not list multiple archives. So
         # blank out any archive filtering flags. They'll break anyway in Borg 2.
-        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_RLIST:
+        for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
             setattr(archive_arguments, name, None)
 
         main_command = make_list_command(
             repository_path,
-            storage_config,
+            config,
             local_borg_version,
             archive_arguments,
             global_arguments,
@@ -250,6 +257,8 @@ def list_archive(
         execute_command(
             main_command,
             output_log_level=logging.ANSWER,
+            environment=environment.make_environment(config),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
-            extra_environment=borg_environment,
+            borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/mount.py

@@ -1,5 +1,6 @@
 import logging
 
+import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
@@ -10,7 +11,7 @@ def mount_archive(
     repository_path,
     archive,
     mount_arguments,
-    storage_config,
+    config,
     local_borg_version,
     global_arguments,
     local_path='borg',
@@ -22,8 +23,8 @@ def mount_archive(
     dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional
     local and remote Borg paths, mount the archive onto the mount point.
     '''
-    umask = storage_config.get('umask', None)
-    lock_wait = storage_config.get('lock_wait', None)
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
 
     full_command = (
         (local_path, 'mount')
@@ -58,16 +59,24 @@ def mount_archive(
         + (tuple(mount_arguments.paths) if mount_arguments.paths else ())
     )
 
-    borg_environment = environment.make_environment(storage_config)
+    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
     if mount_arguments.foreground:
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
+            environment=environment.make_environment(config),
+            working_directory=working_directory,
             borg_local_path=local_path,
-            extra_environment=borg_environment,
+            borg_exit_codes=config.get('borg_exit_codes'),
         )
         return
 
-    execute_command(full_command, borg_local_path=local_path, extra_environment=borg_environment)
+    execute_command(
+        full_command,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/passcommand.py

@@ -0,0 +1,40 @@
+import functools
+import logging
+import shlex
+
+import borgmatic.config.paths
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+@functools.cache
+def run_passcommand(passcommand, working_directory):
+    '''
+    Run the given passcommand using the given working directory and return the passphrase produced
+    by the command.
+
+    Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
+    per borgmatic invocation.
+    '''
+    return borgmatic.execute.execute_command_and_capture_output(
+        shlex.split(passcommand),
+        working_directory=working_directory,
+    )
+
+
+def get_passphrase_from_passcommand(config):
+    '''
+    Given the configuration dict, call the configured passcommand to produce and return an
+    encryption passphrase. In effect, we're doing an end-run around Borg by invoking its passcommand
+    ourselves. This allows us to pass the resulting passphrase to multiple different Borg
+    invocations without the user having to be prompted multiple times.
+
+    If no passcommand is configured, then return None.
+    '''
+    passcommand = config.get('encryption_passcommand')
+
+    if not passcommand:
+        return None
+
+    return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config))

borgmatic/borg/pattern.py

@@ -0,0 +1,50 @@
+import collections
+import enum
+
+
+# See https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns
+class Pattern_type(enum.Enum):
+    ROOT = 'R'  # A ROOT pattern always has a NONE pattern style.
+    PATTERN_STYLE = 'P'
+    EXCLUDE = '-'
+    NO_RECURSE = '!'
+    INCLUDE = '+'
+
+
+class Pattern_style(enum.Enum):
+    NONE = ''
+    FNMATCH = 'fm'
+    SHELL = 'sh'
+    REGULAR_EXPRESSION = 're'
+    PATH_PREFIX = 'pp'
+    PATH_FULL_MATCH = 'pf'
+
+
+class Pattern_source(enum.Enum):
+    '''
+    Where the pattern came from within borgmatic. This is important because certain use cases (like
+    filesystem snapshotting) only want to consider patterns that the user actually put in a
+    configuration file and not patterns from other sources.
+    '''
+
+    # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories".
+    CONFIG = 'config'
+
+    # The pattern is generated internally within borgmatic, e.g. for special file excludes.
+    INTERNAL = 'internal'
+
+    # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump
+    # directory.
+    HOOK = 'hook'
+
+
+Pattern = collections.namedtuple(
+    'Pattern',
+    ('path', 'type', 'style', 'device', 'source'),
+    defaults=(
+        Pattern_type.ROOT,
+        Pattern_style.NONE,
+        None,
+        Pattern_source.HOOK,
+    ),
+)

borgmatic/borg/prune.py

@@ -1,5 +1,6 @@
 import logging
 
+import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command
@@ -7,9 +8,10 @@ from borgmatic.execute import execute_command
 logger = logging.getLogger(__name__)
 
 
-def make_prune_flags(storage_config, retention_config, local_borg_version):
+def make_prune_flags(config, prune_arguments, local_borg_version):
     '''
-    Given a retention config dict mapping from option name to value, transform it into an sequence of
+    Given a configuration dict mapping from option name to value, prune arguments as an
+    argparse.Namespace instance, and the local Borg version, produce a corresponding sequence of
     command-line flags.
 
     For example, given a retention config of:
@@ -23,12 +25,12 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
             ('--keep-monthly', '6'),
         )
     '''
-    config = retention_config.copy()
-    prefix = config.pop('prefix', None)
-
     flag_pairs = (
-        ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
+        ('--' + option_name.replace('_', '-'), str(value))
+        for option_name, value in config.items()
+        if option_name.startswith('keep_') and option_name != 'keep_exclude_tags'
     )
+    prefix = config.get('prefix')
 
     return tuple(element for pair in flag_pairs for element in pair) + (
         (
@@ -39,8 +41,8 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
         if prefix
         else (
             flags.make_match_archives_flags(
-                storage_config.get('match_archives'),
-                storage_config.get('archive_name_format'),
+                prune_arguments.match_archives or config.get('match_archives'),
+                config.get('archive_name_format'),
                 local_borg_version,
             )
         )
@@ -50,8 +52,7 @@ def make_prune_flags(storage_config, retention_config, local_borg_version):
 def prune_archives(
     dry_run,
     repository_path,
-    storage_config,
-    retention_config,
+    config,
     local_borg_version,
     prune_arguments,
     global_arguments,
@@ -59,18 +60,17 @@ def prune_archives(
     remote_path=None,
 ):
     '''
-    Given dry-run flag, a local or remote repository path, a storage config dict, and a
-    retention config dict, prune Borg archives according to the retention policy specified in that
-    configuration.
+    Given dry-run flag, a local or remote repository path, and a configuration dict, prune Borg
+    archives according to the retention policy specified in that configuration.
     '''
     borgmatic.logger.add_custom_log_levels()
-    umask = storage_config.get('umask', None)
-    lock_wait = storage_config.get('lock_wait', None)
-    extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '')
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
+    extra_borg_options = config.get('extra_borg_options', {}).get('prune', '')
 
     full_command = (
         (local_path, 'prune')
-        + make_prune_flags(storage_config, retention_config, local_borg_version)
+        + make_prune_flags(config, prune_arguments, local_borg_version)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--log-json',) if global_arguments.log_json else ())
@@ -79,7 +79,7 @@ def prune_archives(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + flags.make_flags_from_arguments(
             prune_arguments,
-            excludes=('repository', 'stats', 'list_archives'),
+            excludes=('repository', 'match_archives', 'stats', 'list_archives'),
         )
         + (('--list',) if prune_arguments.list_archives else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
@@ -96,6 +96,8 @@ def prune_archives(
     execute_command(
         full_command,
         output_log_level=output_log_level,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
+        borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/rcreate.py

@@ -1,86 +0,0 @@
-import argparse
-import logging
-import subprocess
-
-from borgmatic.borg import environment, feature, flags, rinfo
-from borgmatic.execute import DO_NOT_CAPTURE, execute_command
-
-logger = logging.getLogger(__name__)
-
-
-RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
-
-
-def create_repository(
-    dry_run,
-    repository_path,
-    storage_config,
-    local_borg_version,
-    global_arguments,
-    encryption_mode,
-    source_repository=None,
-    copy_crypt_key=False,
-    append_only=None,
-    storage_quota=None,
-    make_parent_dirs=False,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a dry-run flag, a local or remote repository path, a storage configuration dict, the local
-    Borg version, a Borg encryption mode, the path to another repo whose key material should be
-    reused, whether the repository should be append-only, and the storage quota to use, create the
-    repository. If the repository already exists, then log and skip creation.
-    '''
-    try:
-        rinfo.display_repository_info(
-            repository_path,
-            storage_config,
-            local_borg_version,
-            argparse.Namespace(json=True),
-            global_arguments,
-            local_path,
-            remote_path,
-        )
-        logger.info(f'{repository_path}: Repository already exists. Skipping creation.')
-        return
-    except subprocess.CalledProcessError as error:
-        if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
-            raise
-
-    lock_wait = storage_config.get('lock_wait')
-    extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '')
-
-    rcreate_command = (
-        (local_path,)
-        + (
-            ('rcreate',)
-            if feature.available(feature.Feature.RCREATE, local_borg_version)
-            else ('init',)
-        )
-        + (('--encryption', encryption_mode) if encryption_mode else ())
-        + (('--other-repo', source_repository) if source_repository else ())
-        + (('--copy-crypt-key',) if copy_crypt_key else ())
-        + (('--append-only',) if append_only else ())
-        + (('--storage-quota', storage_quota) if storage_quota else ())
-        + (('--make-parent-dirs',) if make_parent_dirs else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_flags(repository_path, local_borg_version)
-    )
-
-    if dry_run:
-        logging.info(f'{repository_path}: Skipping repository creation (dry run)')
-        return
-
-    # Do not capture output here, so as to support interactive prompts.
-    execute_command(
-        rcreate_command,
-        output_file=DO_NOT_CAPTURE,
-        borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
-    )

borgmatic/borg/repo_create.py

@@ -0,0 +1,105 @@
+import argparse
+import json
+import logging
+import subprocess
+
+import borgmatic.config.paths
+from borgmatic.borg import environment, feature, flags, repo_info
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
+
+logger = logging.getLogger(__name__)
+
+
+REPO_INFO_REPOSITORY_NOT_FOUND_EXIT_CODES = {2, 13, 15}
+
+
+def create_repository(
+    dry_run,
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    encryption_mode,
+    source_repository=None,
+    copy_crypt_key=False,
+    append_only=None,
+    storage_quota=None,
+    make_parent_dirs=False,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, a configuration dict, the local Borg
+    version, a Borg encryption mode, the path to another repo whose key material should be reused,
+    whether the repository should be append-only, and the storage quota to use, create the
+    repository. If the repository already exists, then log and skip creation.
+
+    Raise ValueError if the requested encryption mode does not match that of the repository.
+    Raise json.decoder.JSONDecodeError if the "borg info" JSON outputcannot be decoded.
+    Raise subprocess.CalledProcessError if "borg info" returns an error exit code.
+    '''
+    try:
+        info_data = json.loads(
+            repo_info.display_repository_info(
+                repository_path,
+                config,
+                local_borg_version,
+                argparse.Namespace(json=True),
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        )
+        repository_encryption_mode = info_data.get('encryption', {}).get('mode')
+
+        if repository_encryption_mode != encryption_mode:
+            raise ValueError(
+                f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"'
+            )
+
+        logger.info('Repository already exists. Skipping creation.')
+        return
+    except subprocess.CalledProcessError as error:
+        if error.returncode not in REPO_INFO_REPOSITORY_NOT_FOUND_EXIT_CODES:
+            raise
+
+    lock_wait = config.get('lock_wait')
+    umask = config.get('umask')
+    extra_borg_options = config.get('extra_borg_options', {}).get('repo-create', '')
+
+    repo_create_command = (
+        (local_path,)
+        + (
+            ('repo-create',)
+            if feature.available(feature.Feature.REPO_CREATE, local_borg_version)
+            else ('init',)
+        )
+        + (('--encryption', encryption_mode) if encryption_mode else ())
+        + (('--other-repo', source_repository) if source_repository else ())
+        + (('--copy-crypt-key',) if copy_crypt_key else ())
+        + (('--append-only',) if append_only else ())
+        + (('--storage-quota', storage_quota) if storage_quota else ())
+        + (('--make-parent-dirs',) if make_parent_dirs else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + flags.make_repository_flags(repository_path, local_borg_version)
+    )
+
+    if dry_run:
+        logging.info('Skipping repository creation (dry run)')
+        return
+
+    # Do not capture output here, so as to support interactive prompts.
+    execute_command(
+        repo_create_command,
+        output_file=DO_NOT_CAPTURE,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/repo_delete.py

@@ -0,0 +1,95 @@
+import logging
+
+import borgmatic.borg.environment
+import borgmatic.borg.feature
+import borgmatic.borg.flags
+import borgmatic.config.paths
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def make_repo_delete_command(
+    repository,
+    config,
+    local_borg_version,
+    repo_delete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the repo_delete action as an argparse.Namespace, and global arguments, return a command
+    as a tuple to repo_delete the entire repository.
+    '''
+    return (
+        (local_path,)
+        + (
+            ('repo-delete',)
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+            )
+            else ('delete',)
+        )
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
+        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
+        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
+        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+        + borgmatic.borg.flags.make_flags('list', repo_delete_arguments.list_archives)
+        + (
+            (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ()))
+            if repo_delete_arguments.force
+            else ()
+        )
+        + borgmatic.borg.flags.make_flags_from_arguments(
+            repo_delete_arguments, excludes=('list_archives', 'force', 'repository')
+        )
+        + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
+    )
+
+
+def delete_repository(
+    repository,
+    config,
+    local_borg_version,
+    repo_delete_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the repo_delete action as an argparse.Namespace, global arguments as an
+    argparse.Namespace, and local and remote Borg paths, repo_delete the entire repository.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    command = make_repo_delete_command(
+        repository,
+        config,
+        local_borg_version,
+        repo_delete_arguments,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.execute.execute_command(
+        command,
+        output_log_level=logging.ANSWER,
+        # Don't capture output when Borg is expected to prompt for interactive confirmation, or the
+        # prompt won't work.
+        output_file=(
+            None
+            if repo_delete_arguments.force or repo_delete_arguments.cache_only
+            else borgmatic.execute.DO_NOT_CAPTURE
+        ),
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/repo_info.py

@@ -1,5 +1,6 @@
 import logging
 
+import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command, execute_command_and_capture_output
@@ -9,56 +10,63 @@ logger = logging.getLogger(__name__)
 
 def display_repository_info(
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
-    rinfo_arguments,
+    repo_info_arguments,
     global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary
+    Given a local or remote repository path, a configuration dict, the local Borg version, the
+    arguments to the repo_info action, and global arguments as an argparse.Namespace, display summary
     information for the Borg repository or return JSON summary information.
     '''
     borgmatic.logger.add_custom_log_levels()
-    lock_wait = storage_config.get('lock_wait', None)
+    lock_wait = config.get('lock_wait', None)
 
     full_command = (
         (local_path,)
         + (
-            ('rinfo',)
-            if feature.available(feature.Feature.RINFO, local_borg_version)
+            ('repo-info',)
+            if feature.available(feature.Feature.REPO_INFO, local_borg_version)
             else ('info',)
         )
         + (
             ('--info',)
-            if logger.getEffectiveLevel() == logging.INFO and not rinfo_arguments.json
+            if logger.getEffectiveLevel() == logging.INFO and not repo_info_arguments.json
             else ()
         )
         + (
             ('--debug', '--show-rc')
-            if logger.isEnabledFor(logging.DEBUG) and not rinfo_arguments.json
+            if logger.isEnabledFor(logging.DEBUG) and not repo_info_arguments.json
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
-        + (('--json',) if rinfo_arguments.json else ())
+        + (('--json',) if repo_info_arguments.json else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    extra_environment = environment.make_environment(storage_config)
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+    borg_exit_codes = config.get('borg_exit_codes')
 
-    if rinfo_arguments.json:
+    if repo_info_arguments.json:
         return execute_command_and_capture_output(
             full_command,
-            extra_environment=extra_environment,
+            environment=environment.make_environment(config),
+            working_directory=working_directory,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
         )
     else:
         execute_command(
             full_command,
             output_log_level=logging.ANSWER,
+            environment=environment.make_environment(config),
+            working_directory=working_directory,
             borg_local_path=local_path,
-            extra_environment=extra_environment,
+            borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/repo_list.py

@@ -0,0 +1,185 @@
+import argparse
+import logging
+
+import borgmatic.config.paths
+import borgmatic.logger
+from borgmatic.borg import environment, feature, flags
+from borgmatic.execute import execute_command, execute_command_and_capture_output
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_archive_name(
+    repository_path,
+    archive,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, an archive name, a configuration dict, the local Borg
+    version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
+    return the archive name. But if the archive name is "latest", then instead introspect the
+    repository for the latest archive and return its name.
+
+    Raise ValueError if "latest" is given but there are no archives in the repository.
+    '''
+    if archive != 'latest':
+        return archive
+
+    full_command = (
+        (
+            local_path,
+            (
+                'repo-list'
+                if feature.available(feature.Feature.REPO_LIST, local_borg_version)
+                else 'list'
+            ),
+        )
+        + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('umask', config.get('umask'))
+        + flags.make_flags('log-json', global_arguments.log_json)
+        + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + flags.make_flags('last', 1)
+        + ('--short',)
+        + flags.make_repository_flags(repository_path, local_borg_version)
+    )
+
+    output = execute_command_and_capture_output(
+        full_command,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )
+    try:
+        latest_archive = output.strip().splitlines()[-1]
+    except IndexError:
+        raise ValueError('No archives found in the repository')
+
+    logger.debug(f'Latest archive is {latest_archive}')
+
+    return latest_archive
+
+
+MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
+
+
+def make_repo_list_command(
+    repository_path,
+    config,
+    local_borg_version,
+    repo_list_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, the
+    arguments to the repo_list action, global arguments as an argparse.Namespace instance, and local and
+    remote Borg paths, return a command as a tuple to list archives with a repository.
+    '''
+    return (
+        (
+            local_path,
+            (
+                'repo-list'
+                if feature.available(feature.Feature.REPO_LIST, local_borg_version)
+                else 'list'
+            ),
+        )
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not repo_list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not repo_list_arguments.json
+            else ()
+        )
+        + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('umask', config.get('umask'))
+        + flags.make_flags('log-json', global_arguments.log_json)
+        + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + (
+            (
+                flags.make_flags('match-archives', f'sh:{repo_list_arguments.prefix}*')
+                if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
+                else flags.make_flags('glob-archives', f'{repo_list_arguments.prefix}*')
+            )
+            if repo_list_arguments.prefix
+            else (
+                flags.make_match_archives_flags(
+                    repo_list_arguments.match_archives or config.get('match_archives'),
+                    config.get('archive_name_format'),
+                    local_borg_version,
+                )
+            )
+        )
+        + flags.make_flags_from_arguments(repo_list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
+        + flags.make_repository_flags(repository_path, local_borg_version)
+    )
+
+
+def list_repository(
+    repository_path,
+    config,
+    local_borg_version,
+    repo_list_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, the
+    arguments to the list action, global arguments as an argparse.Namespace instance, and local and
+    remote Borg paths, display the output of listing Borg archives in the given repository (or
+    return JSON output).
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    main_command = make_repo_list_command(
+        repository_path,
+        config,
+        local_borg_version,
+        repo_list_arguments,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    json_command = make_repo_list_command(
+        repository_path,
+        config,
+        local_borg_version,
+        argparse.Namespace(**dict(repo_list_arguments.__dict__, json=True)),
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+    borg_exit_codes = config.get('borg_exit_codes')
+
+    json_listing = execute_command_and_capture_output(
+        json_command,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )
+
+    if repo_list_arguments.json:
+        return json_listing
+
+    flags.warn_for_aggressive_archive_flags(json_command, json_listing)
+
+    execute_command(
+        main_command,
+        output_log_level=logging.ANSWER,
+        environment=environment.make_environment(config),
+        working_directory=working_directory,
+        borg_local_path=local_path,
+        borg_exit_codes=borg_exit_codes,
+    )

borgmatic/borg/rlist.py

@@ -1,148 +0,0 @@
-import logging
-
-import borgmatic.logger
-from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command, execute_command_and_capture_output
-
-logger = logging.getLogger(__name__)
-
-
-def resolve_archive_name(
-    repository_path,
-    archive,
-    storage_config,
-    local_borg_version,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, an archive name, a storage config dict, the local Borg
-    version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
-    return the archive name. But if the archive name is "latest", then instead introspect the
-    repository for the latest archive and return its name.
-
-    Raise ValueError if "latest" is given but there are no archives in the repository.
-    '''
-    if archive != 'latest':
-        return archive
-
-    full_command = (
-        (
-            local_path,
-            'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list',
-        )
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', storage_config.get('lock_wait'))
-        + flags.make_flags('last', 1)
-        + ('--short',)
-        + flags.make_repository_flags(repository_path, local_borg_version)
-    )
-
-    output = execute_command_and_capture_output(
-        full_command,
-        extra_environment=environment.make_environment(storage_config),
-    )
-    try:
-        latest_archive = output.strip().splitlines()[-1]
-    except IndexError:
-        raise ValueError('No archives found in the repository')
-
-    logger.debug(f'{repository_path}: Latest archive is {latest_archive}')
-
-    return latest_archive
-
-
-MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
-
-
-def make_rlist_command(
-    repository_path,
-    storage_config,
-    local_borg_version,
-    rlist_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and
-    remote Borg paths, return a command as a tuple to list archives with a repository.
-    '''
-    return (
-        (
-            local_path,
-            'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list',
-        )
-        + (
-            ('--info',)
-            if logger.getEffectiveLevel() == logging.INFO and not rlist_arguments.json
-            else ()
-        )
-        + (
-            ('--debug', '--show-rc')
-            if logger.isEnabledFor(logging.DEBUG) and not rlist_arguments.json
-            else ()
-        )
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', storage_config.get('lock_wait'))
-        + (
-            (
-                flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*')
-                if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
-                else flags.make_flags('glob-archives', f'{rlist_arguments.prefix}*')
-            )
-            if rlist_arguments.prefix
-            else (
-                flags.make_match_archives_flags(
-                    rlist_arguments.match_archives or storage_config.get('match_archives'),
-                    storage_config.get('archive_name_format'),
-                    local_borg_version,
-                )
-            )
-        )
-        + flags.make_flags_from_arguments(rlist_arguments, excludes=MAKE_FLAGS_EXCLUDES)
-        + flags.make_repository_flags(repository_path, local_borg_version)
-    )
-
-
-def list_repository(
-    repository_path,
-    storage_config,
-    local_borg_version,
-    rlist_arguments,
-    global_arguments,
-    local_path='borg',
-    remote_path=None,
-):
-    '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the list action, global arguments as an argparse.Namespace instance, and local and
-    remote Borg paths, display the output of listing Borg archives in the given repository (or
-    return JSON output).
-    '''
-    borgmatic.logger.add_custom_log_levels()
-    borg_environment = environment.make_environment(storage_config)
-
-    main_command = make_rlist_command(
-        repository_path,
-        storage_config,
-        local_borg_version,
-        rlist_arguments,
-        global_arguments,
-        local_path,
-        remote_path,
-    )
-
-    if rlist_arguments.json:
-        return execute_command_and_capture_output(main_command, extra_environment=borg_environment)
-    else:
-        execute_command(
-            main_command,
-            output_log_level=logging.ANSWER,
-            borg_local_path=local_path,
-            extra_environment=borg_environment,
-        )

borgmatic/borg/transfer.py

@@ -1,5 +1,6 @@
 import logging
 
+import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
@@ -10,7 +11,7 @@ logger = logging.getLogger(__name__)
 def transfer_archives(
     dry_run,
     repository_path,
-    storage_config,
+    config,
     local_borg_version,
     transfer_arguments,
     global_arguments,
@@ -18,7 +19,7 @@ def transfer_archives(
     remote_path=None,
 ):
     '''
-    Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg
+    Given a dry-run flag, a local or remote repository path, a configuration dict, the local Borg
     version, the arguments to the transfer action, and global arguments as an argparse.Namespace
     instance, transfer archives to the given repository.
     '''
@@ -29,8 +30,9 @@ def transfer_archives(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', global_arguments.log_json)
-        + flags.make_flags('lock-wait', storage_config.get('lock_wait', None))
+        + flags.make_flags('lock-wait', config.get('lock_wait', None))
         + (
             flags.make_flags_from_arguments(
                 transfer_arguments,
@@ -40,8 +42,8 @@ def transfer_archives(
                 flags.make_match_archives_flags(
                     transfer_arguments.match_archives
                     or transfer_arguments.archive
-                    or storage_config.get('match_archives'),
-                    storage_config.get('archive_name_format'),
+                    or config.get('match_archives'),
+                    config.get('archive_name_format'),
                     local_borg_version,
                 )
             )
@@ -55,6 +57,8 @@ def transfer_archives(
         full_command,
         output_log_level=logging.ANSWER,
         output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
+        borg_exit_codes=config.get('borg_exit_codes'),
     )

borgmatic/borg/umount.py

@@ -1,11 +1,12 @@
 import logging
 
+import borgmatic.config.paths
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def unmount_archive(mount_point, local_path='borg'):
+def unmount_archive(config, mount_point, local_path='borg'):
     '''
     Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
     from the mount point.
@@ -17,4 +18,9 @@ def unmount_archive(mount_point, local_path='borg'):
         + (mount_point,)
     )
 
-    execute_command(full_command)
+    execute_command(
+        full_command,
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/version.py

@@ -1,14 +1,15 @@
 import logging
 
+import borgmatic.config.paths
 from borgmatic.borg import environment
 from borgmatic.execute import execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
 
-def local_borg_version(storage_config, local_path='borg'):
+def local_borg_version(config, local_path='borg'):
     '''
-    Given a storage configuration dict and a local Borg binary path, return a version string for it.
+    Given a configuration dict and a local Borg executable path, return a version string for it.
 
     Raise OSError or CalledProcessError if there is a problem running Borg.
     Raise ValueError if the version cannot be parsed.
@@ -20,7 +21,10 @@ def local_borg_version(storage_config, local_path='borg'):
     )
     output = execute_command_and_capture_output(
         full_command,
-        extra_environment=environment.make_environment(storage_config),
+        environment=environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
     )
 
     try:

borgmatic/commands/completion/actions.py

@@ -0,0 +1,36 @@
+import borgmatic.commands.arguments
+
+
+def upgrade_message(language: str, upgrade_command: str, completion_file: str):
+    return f'''
+Your {language} completions script is from a different version of borgmatic than is
+currently installed. Please upgrade your script so your completions match the
+command-line flags in your installed borgmatic! Try this to upgrade:
+
+    {upgrade_command}
+    source {completion_file}
+'''
+
+
+def available_actions(subparsers, current_action=None):
+    '''
+    Given subparsers as an argparse._SubParsersAction instance and a current action name (if
+    any), return the actions names that can follow the current action on a command-line.
+
+    This takes into account which sub-actions that the current action supports. For instance, if
+    "bootstrap" is a sub-action for "config", then "bootstrap" should be able to follow a current
+    action of "config" but not "list".
+    '''
+    action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions(
+        subparsers.choices
+    )
+    current_subactions = action_to_subactions.get(current_action)
+
+    if current_subactions:
+        return current_subactions
+
+    all_subactions = set(
+        subaction for subactions in action_to_subactions.values() for subaction in subactions
+    )
+
+    return tuple(action for action in subparsers.choices.keys() if action not in all_subactions)

borgmatic/commands/completion/bash.py

@@ -0,0 +1,66 @@
+import borgmatic.commands.arguments
+import borgmatic.commands.completion.actions
+
+
+def parser_flags(parser):
+    '''
+    Given an argparse.ArgumentParser instance, return its argument flags in a space-separated
+    string.
+    '''
+    return ' '.join(option for action in parser._actions for option in action.option_strings)
+
+
+def bash_completion():
+    '''
+    Return a bash completion script for the borgmatic command. Produce this by introspecting
+    borgmatic's command-line argument parsers.
+    '''
+    (
+        unused_global_parser,
+        action_parsers,
+        global_plus_action_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
+    global_flags = parser_flags(global_plus_action_parser)
+
+    # Avert your eyes.
+    return '\n'.join(
+        (
+            'check_version() {',
+            '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
+            '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
+            '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
+            f'''        then cat << EOF\n{borgmatic.commands.completion.actions.upgrade_message(
+                    'bash',
+                    'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
+                    '$BASH_SOURCE',
+                )}\nEOF''',
+            '    fi',
+            '}',
+            'complete_borgmatic() {',
+        )
+        + tuple(
+            '''    if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
+        COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
+        return 0
+    fi'''
+            % (
+                action,
+                parser_flags(action_parser),
+                ' '.join(
+                    borgmatic.commands.completion.actions.available_actions(action_parsers, action)
+                ),
+                global_flags,
+            )
+            for action, action_parser in reversed(action_parsers.choices.items())
+        )
+        + (
+            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
+            % (
+                ' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)),
+                global_flags,
+            ),
+            '    (check_version &)',
+            '}',
+            '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
+        )
+    )

borgmatic/commands/completion/fish.py

@@ -2,72 +2,8 @@ import shlex
 from argparse import Action
 from textwrap import dedent
 
-from borgmatic.commands import arguments
-
-
-def upgrade_message(language: str, upgrade_command: str, completion_file: str):
-    return f'''
-Your {language} completions script is from a different version of borgmatic than is
-currently installed. Please upgrade your script so your completions match the
-command-line flags in your installed borgmatic! Try this to upgrade:
-
-    {upgrade_command}
-    source {completion_file}
-'''
-
-
-def parser_flags(parser):
-    '''
-    Given an argparse.ArgumentParser instance, return its argument flags in a space-separated
-    string.
-    '''
-    return ' '.join(option for action in parser._actions for option in action.option_strings)
-
-
-def bash_completion():
-    '''
-    Return a bash completion script for the borgmatic command. Produce this by introspecting
-    borgmatic's command-line argument parsers.
-    '''
-    top_level_parser, subparsers = arguments.make_parsers()
-    global_flags = parser_flags(top_level_parser)
-    actions = ' '.join(subparsers.choices.keys())
-
-    # Avert your eyes.
-    return '\n'.join(
-        (
-            'check_version() {',
-            '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
-            '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
-            '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
-            f'''        then cat << EOF\n{upgrade_message(
-                    'bash',
-                    'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
-                    '$BASH_SOURCE',
-                )}\nEOF''',
-            '    fi',
-            '}',
-            'complete_borgmatic() {',
-        )
-        + tuple(
-            '''    if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
-        COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
-        return 0
-    fi'''
-            % (action, parser_flags(subparser), actions, global_flags)
-            for action, subparser in subparsers.choices.items()
-        )
-        + (
-            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
-            % (actions, global_flags),
-            '    (check_version &)',
-            '}',
-            '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
-        )
-    )
-
-
-# fish section
+import borgmatic.commands.arguments
+import borgmatic.commands.completion.actions
 
 
 def has_file_options(action: Action):
@@ -155,18 +91,22 @@ def fish_completion():
     Return a fish completion script for the borgmatic command. Produce this by introspecting
     borgmatic's command-line argument parsers.
     '''
-    top_level_parser, subparsers = arguments.make_parsers()
+    (
+        unused_global_parser,
+        action_parsers,
+        global_plus_action_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
 
-    all_subparsers = ' '.join(action for action in subparsers.choices.keys())
+    all_action_parsers = ' '.join(action for action in action_parsers.choices.keys())
 
     exact_option_args = tuple(
         ' '.join(action.option_strings)
-        for subparser in subparsers.choices.values()
-        for action in subparser._actions
+        for action_parser in action_parsers.choices.values()
+        for action in action_parser._actions
         if has_exact_options(action)
     ) + tuple(
         ' '.join(action.option_strings)
-        for action in top_level_parser._actions
+        for action in global_plus_action_parser._actions
         if len(action.option_strings) > 0
         if has_exact_options(action)
     )
@@ -182,7 +122,7 @@ def fish_completion():
                         set this_script (cat $this_filename 2> /dev/null)
                         set installed_script (borgmatic --fish-completion 2> /dev/null)
                         if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]
-                            echo "{upgrade_message(
+                            echo "{borgmatic.commands.completion.actions.upgrade_message(
                             'fish',
                             'borgmatic --fish-completion | sudo tee $this_filename',
                             '$this_filename',
@@ -208,29 +148,29 @@ def fish_completion():
                 return 1
             end
 
-            set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"
+            set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}"
             set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
             '''
         )
-        + ('\n# subparser completions',)
+        + ('\n# action_parser completions',)
         + tuple(
-            f'''complete -c borgmatic -f -n "$subparser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}'''
-            for action_name, subparser in subparsers.choices.items()
+            f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}'''
+            for action_name, action_parser in action_parsers.choices.items()
         )
         + ('\n# global flags',)
         + tuple(
             # -n is checked in order, so put faster / more likely to be true checks first
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
-            for action in top_level_parser._actions
+            for action in global_plus_action_parser._actions
             # ignore the noargs action, as this is an impossible completion for fish
             if len(action.option_strings) > 0
             if 'Deprecated' not in action.help
         )
-        + ('\n# subparser flags',)
+        + ('\n# action_parser flags',)
         + tuple(
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
-            for action_name, subparser in subparsers.choices.items()
-            for action in subparser._actions
-            if 'Deprecated' not in action.help
+            for action_name, action_parser in action_parsers.choices.items()
+            for action in action_parser._actions
+            if 'Deprecated' not in (action.help or ())
         )
     )

borgmatic/commands/convert_config.py

@@ -1,102 +0,0 @@
-import os
-import sys
-import textwrap
-from argparse import ArgumentParser
-
-from ruamel import yaml
-
-from borgmatic.config import convert, generate, legacy, validate
-
-DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
-DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
-DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
-
-
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    parser = ArgumentParser(
-        description='''
-            Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
-            configuration file. Note that this replaces any comments from the source files.
-        '''
-    )
-    parser.add_argument(
-        '-s',
-        '--source-config',
-        dest='source_config_filename',
-        default=DEFAULT_SOURCE_CONFIG_FILENAME,
-        help=f'Source INI-style configuration filename. Default: {DEFAULT_SOURCE_CONFIG_FILENAME}',
-    )
-    parser.add_argument(
-        '-e',
-        '--source-excludes',
-        dest='source_excludes_filename',
-        default=DEFAULT_SOURCE_EXCLUDES_FILENAME
-        if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME)
-        else None,
-        help='Excludes filename',
-    )
-    parser.add_argument(
-        '-d',
-        '--destination-config',
-        dest='destination_config_filename',
-        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help=f'Destination YAML configuration filename. Default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
-    )
-
-    return parser.parse_args(arguments)
-
-
-TEXT_WRAP_CHARACTERS = 80
-
-
-def display_result(args):  # pragma: no cover
-    result_lines = textwrap.wrap(
-        f'Your borgmatic configuration has been upgraded. Please review the result in {args.destination_config_filename}.',
-        TEXT_WRAP_CHARACTERS,
-    )
-
-    excludes_phrase = (
-        f' and {args.source_excludes_filename}' if args.source_excludes_filename else ''
-    )
-    delete_lines = textwrap.wrap(
-        f'Once you are satisfied, you can safely delete {args.source_config_filename}{excludes_phrase}.',
-        TEXT_WRAP_CHARACTERS,
-    )
-
-    print('\n'.join(result_lines))
-    print()
-    print('\n'.join(delete_lines))
-
-
-def main():  # pragma: no cover
-    try:
-        args = parse_arguments(*sys.argv[1:])
-        schema = yaml.round_trip_load(open(validate.schema_filename()).read())
-        source_config = legacy.parse_configuration(
-            args.source_config_filename, legacy.CONFIG_FORMAT
-        )
-        source_config_file_mode = os.stat(args.source_config_filename).st_mode
-        source_excludes = (
-            open(args.source_excludes_filename).read().splitlines()
-            if args.source_excludes_filename
-            else []
-        )
-
-        destination_config = convert.convert_legacy_parsed_config(
-            source_config, source_excludes, schema
-        )
-
-        generate.write_configuration(
-            args.destination_config_filename,
-            generate.render_configuration(destination_config),
-            mode=source_config_file_mode,
-        )
-
-        display_result(args)
-    except (ValueError, OSError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)

borgmatic/commands/generate_config.py

@@ -1,63 +1,17 @@
+import logging
 import sys
-from argparse import ArgumentParser
 
-from borgmatic.config import generate, validate
-
-DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+import borgmatic.commands.borgmatic
 
 
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
-    parser.add_argument(
-        '-s',
-        '--source',
-        dest='source_filename',
-        help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration',
-    )
-    parser.add_argument(
-        '-d',
-        '--destination',
-        dest='destination_filename',
-        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
-    )
-    parser.add_argument(
-        '--overwrite',
-        default=False,
-        action='store_true',
-        help='Whether to overwrite any existing destination file, defaults to false',
-    )
-
-    return parser.parse_args(arguments)
-
-
-def main():  # pragma: no cover
-    try:
-        args = parse_arguments(*sys.argv[1:])
-
-        generate.generate_sample_configuration(
-            args.source_filename,
-            args.destination_filename,
-            validate.schema_filename(),
-            overwrite=args.overwrite,
+def main():
+    warning_log = logging.makeLogRecord(
+        dict(
+            levelno=logging.WARNING,
+            levelname='WARNING',
+            msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
         )
+    )
 
-        print(f'Generated a sample configuration file at {args.destination_filename}.')
-        print()
-        if args.source_filename:
-            print(f'Merged in the contents of configuration file at {args.source_filename}.')
-            print('To review the changes made, run:')
-            print()
-            print(f'    diff --unified {args.source_filename} {args.destination_filename}')
-            print()
-        print('This includes all available configuration options with example values. The few')
-        print('required options are indicated. Please edit the file to suit your needs.')
-        print()
-        print('If you ever need help: https://torsion.org/borgmatic/#issues')
-    except (ValueError, OSError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)
+    sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
+    borgmatic.commands.borgmatic.main([warning_log])

borgmatic/commands/validate_config.py

@@ -1,68 +1,17 @@
 import logging
 import sys
-from argparse import ArgumentParser
 
-import borgmatic.config.generate
-from borgmatic.config import collect, validate
-
-logger = logging.getLogger(__name__)
+import borgmatic.commands.borgmatic
 
 
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    config_paths = collect.get_default_config_paths()
-
-    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
-    parser.add_argument(
-        '-c',
-        '--config',
-        nargs='+',
-        dest='config_paths',
-        default=config_paths,
-        help=f'Configuration filenames or directories, defaults to: {config_paths}',
-    )
-    parser.add_argument(
-        '-s',
-        '--show',
-        action='store_true',
-        help='Show the validated configuration after all include merging has occurred',
+def main():
+    warning_log = logging.makeLogRecord(
+        dict(
+            levelno=logging.WARNING,
+            levelname='WARNING',
+            msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.',
+        )
     )
 
-    return parser.parse_args(arguments)
-
-
-def main():  # pragma: no cover
-    arguments = parse_arguments(*sys.argv[1:])
-
-    logging.basicConfig(level=logging.INFO, format='%(message)s')
-
-    config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths))
-    if len(config_filenames) == 0:
-        logger.critical('No files to validate found')
-        sys.exit(1)
-
-    found_issues = False
-    for config_filename in config_filenames:
-        try:
-            config, parse_logs = validate.parse_configuration(
-                config_filename, validate.schema_filename()
-            )
-        except (ValueError, OSError, validate.Validation_error) as error:
-            logging.critical(f'{config_filename}: Error parsing configuration file')
-            logging.critical(error)
-            found_issues = True
-        else:
-            for log in parse_logs:
-                logger.handle(log)
-
-            if arguments.show:
-                print('---')
-                print(borgmatic.config.generate.render_configuration(config))
-
-    if found_issues:
-        sys.exit(1)
-
-    logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")
+    sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:]
+    borgmatic.commands.borgmatic.main([warning_log])

borgmatic/config/checks.py

@@ -1,9 +1,9 @@
-def repository_enabled_for_checks(repository, consistency):
+def repository_enabled_for_checks(repository, config):
     '''
-    Given a repository name and a consistency configuration dict, return whether the repository
-    is enabled to have consistency checks run.
+    Given a repository name and a configuration dict, return whether the
+    repository is enabled to have consistency checks run.
     '''
-    if not consistency.get('check_repositories'):
+    if not config.get('check_repositories'):
         return True
 
-    return repository in consistency['check_repositories']
+    return repository in config['check_repositories']

borgmatic/config/collect.py

@@ -24,9 +24,9 @@ def get_default_config_paths(expand_home=True):
 def collect_config_filenames(config_paths):
     '''
     Given a sequence of config paths, both filenames and directories, resolve that to an iterable
-    of files. Accomplish this by listing any given directories looking for contained config files
-    (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given
-    directories are ignored.
+    of absolute files. Accomplish this by listing any given directories looking for contained config
+    files (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories
+    within the given directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
     configuration paths. However, skip a default config path if it's missing, so the user doesn't
@@ -41,7 +41,7 @@ def collect_config_filenames(config_paths):
             continue
 
         if not os.path.isdir(path) or not exists:
-            yield path
+            yield os.path.abspath(path)
             continue
 
         if not os.access(path, os.R_OK):
@@ -51,4 +51,4 @@ def collect_config_filenames(config_paths):
             full_filename = os.path.join(path, filename)
             matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
             if matching_filetype and not os.path.isdir(full_filename):
-                yield full_filename
+                yield os.path.abspath(full_filename)

borgmatic/config/constants.py

@@ -0,0 +1,64 @@
+import shlex
+
+
+def coerce_scalar(value):
+    '''
+    Given a configuration value, coerce it to an integer or a boolean as appropriate and return the
+    result.
+    '''
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        pass
+
+    if value == 'true' or value == 'True':
+        return True
+    if value == 'false' or value == 'False':
+        return False
+
+    return value
+
+
+def apply_constants(value, constants, shell_escape=False):
+    '''
+    Given a configuration value (bool, dict, int, list, or string) and a dict of named constants,
+    replace any configuration string values of the form "{constant}" (or containing it) with the
+    value of the correspondingly named key from the constants. Recurse as necessary into nested
+    configuration to find values to replace.
+
+    For instance, if a configuration value contains "{foo}", replace it with the value of the "foo"
+    key found within the configuration's "constants".
+
+    If shell escape is True, then escape the constant's value before applying it.
+
+    Return the configuration value and modify the original.
+    '''
+    if not value or not constants:
+        return value
+
+    if isinstance(value, str):
+        for constant_name, constant_value in constants.items():
+            value = value.replace(
+                '{' + constant_name + '}',
+                shlex.quote(str(constant_value)) if shell_escape else str(constant_value),
+            )
+
+        # Support constants within non-string scalars by coercing the value to its appropriate type.
+        value = coerce_scalar(value)
+    elif isinstance(value, list):
+        for index, list_value in enumerate(value):
+            value[index] = apply_constants(list_value, constants, shell_escape)
+    elif isinstance(value, dict):
+        for option_name, option_value in value.items():
+            value[option_name] = apply_constants(
+                option_value,
+                constants,
+                shell_escape=(
+                    shell_escape
+                    or option_name.startswith('before_')
+                    or option_name.startswith('after_')
+                    or option_name == 'on_error'
+                ),
+            )
+
+    return value

borgmatic/config/convert.py

@@ -1,95 +0,0 @@
-import os
-
-from ruamel import yaml
-
-from borgmatic.config import generate
-
-
-def _convert_section(source_section_config, section_schema):
-    '''
-    Given a legacy Parsed_config instance for a single section, convert it to its corresponding
-    yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
-
-    Where integer types exist in the given section schema, convert their values to integers.
-    '''
-    destination_section_config = yaml.comments.CommentedMap(
-        [
-            (
-                option_name,
-                int(option_value)
-                if section_schema['properties'].get(option_name, {}).get('type') == 'integer'
-                else option_value,
-            )
-            for option_name, option_value in source_section_config.items()
-        ]
-    )
-
-    return destination_section_config
-
-
-def convert_legacy_parsed_config(source_config, source_excludes, schema):
-    '''
-    Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
-    patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
-    preparation for serialization to a single YAML config file.
-
-    Additionally, use the given schema as a source of helpful comments to include within the
-    returned CommentedMap.
-    '''
-    destination_config = yaml.comments.CommentedMap(
-        [
-            (section_name, _convert_section(section_config, schema['properties'][section_name]))
-            for section_name, section_config in source_config._asdict().items()
-        ]
-    )
-
-    # Split space-separated values into actual lists, make "repository" into a list, and merge in
-    # excludes.
-    location = destination_config['location']
-    location['source_directories'] = source_config.location['source_directories'].split(' ')
-    location['repositories'] = [location.pop('repository')]
-    location['exclude_patterns'] = source_excludes
-
-    if source_config.consistency.get('checks'):
-        destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
-
-    # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration_object(destination_config, schema)
-
-    for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration_object(
-            section_config, schema['properties'][section_name], indent=generate.INDENT
-        )
-
-    return destination_config
-
-
-class Legacy_configuration_not_upgraded(FileNotFoundError):
-    def __init__(self):
-        super(Legacy_configuration_not_upgraded, self).__init__(
-            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
-to YAML. This better supports validation, and has a more natural way to express
-lists of values. To upgrade your existing configuration, run:
-
-    sudo upgrade-borgmatic-config
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and excludes
-files. The new version of borgmatic will consume the YAML configuration file
-instead of the old one.'''
-        )
-
-
-def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
-    '''
-    If legacy source configuration exists but no destination upgraded configs do, raise
-    Legacy_configuration_not_upgraded.
-
-    The idea is that we want to alert the user about upgrading their config if they haven't already.
-    '''
-    destination_config_exists = any(
-        os.path.exists(filename) for filename in destination_config_filenames
-    )
-
-    if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise Legacy_configuration_not_upgraded()

borgmatic/config/environment.py

@@ -1,21 +1,22 @@
 import os
 import re
 
-_VARIABLE_PATTERN = re.compile(
+VARIABLE_PATTERN = re.compile(
     r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
 )
 
 
-def _resolve_string(matcher):
+def resolve_string(matcher):
     '''
-    Get the value from environment given a matcher containing a name and an optional default value.
-    If the variable is not defined in environment and no default value is provided, an Error is raised.
+    Given a matcher containing a name and an optional default value, get the value from environment.
+
+    Raise ValueError if the variable is not defined in environment and no default value is provided.
     '''
     if matcher.group('escape') is not None:
-        # in case of escaped envvar, unescape it
+        # In the case of an escaped environment variable, unescape it.
         return matcher.group('variable')
 
-    # resolve the env var
+    # Resolve the environment variable.
     name, default = matcher.group('name'), matcher.group('default')
     out = os.getenv(name, default=default)
 
@@ -27,19 +28,24 @@ def _resolve_string(matcher):
 
 def resolve_env_variables(item):
     '''
-    Resolves variables like or ${FOO} from given configuration with values from process environment
-    Supported formats:
-     - ${FOO} will return FOO env variable
-     - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
+    Resolves variables like or ${FOO} from given configuration with values from process environment.
 
-    If any variable is missing in environment and no default value is provided, an Error is raised.
+    Supported formats:
+
+     * ${FOO} will return FOO env variable
+     * ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
+
+    Raise if any variable is missing in environment and no default value is provided.
     '''
     if isinstance(item, str):
-        return _VARIABLE_PATTERN.sub(_resolve_string, item)
+        return VARIABLE_PATTERN.sub(resolve_string, item)
+
     if isinstance(item, list):
-        for i, subitem in enumerate(item):
-            item[i] = resolve_env_variables(subitem)
+        for index, subitem in enumerate(item):
+            item[index] = resolve_env_variables(subitem)
+
     if isinstance(item, dict):
         for key, value in item.items():
             item[key] = resolve_env_variables(value)
+
     return item

borgmatic/config/generate.py

@@ -3,7 +3,7 @@ import io
 import os
 import re
 
-from ruamel import yaml
+import ruamel.yaml
 
 from borgmatic.config import load, normalize
 
@@ -11,36 +11,50 @@ INDENT = 4
 SEQUENCE_INDENT = 2
 
 
-def _insert_newline_before_comment(config, field_name):
+def insert_newline_before_comment(config, field_name):
     '''
     Using some ruamel.yaml black magic, insert a blank line in the config right before the given
     field and its comments.
     '''
     config.ca.items[field_name][1].insert(
-        0, yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None)
+        0, ruamel.yaml.tokens.CommentToken('\n', ruamel.yaml.error.CommentMark(0), None)
     )
 
 
-def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
+def get_properties(schema):
+    '''
+    Given a schema dict, return its properties. But if it's got sub-schemas with multiple different
+    potential properties, returned their merged properties instead.
+    '''
+    if 'oneOf' in schema:
+        return dict(
+            collections.ChainMap(*[sub_schema['properties'] for sub_schema in schema['oneOf']])
+        )
+
+    return schema['properties']
+
+
+def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema, generate and return sample config for it. Include comments
-    for each section based on the schema "description".
+    for each option based on the schema "description".
     '''
     schema_type = schema.get('type')
     example = schema.get('example')
+
     if example is not None:
         return example
 
-    if schema_type == 'array':
-        config = yaml.comments.CommentedSeq(
-            [_schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
+    if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type):
+        config = ruamel.yaml.comments.CommentedSeq(
+            [schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
         )
         add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
-    elif schema_type == 'object':
-        config = yaml.comments.CommentedMap(
+    elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type):
+        config = ruamel.yaml.comments.CommentedMap(
             [
-                (field_name, _schema_to_sample_configuration(sub_schema, level + 1))
-                for field_name, sub_schema in schema['properties'].items()
+                (field_name, schema_to_sample_configuration(sub_schema, level + 1))
+                for field_name, sub_schema in get_properties(schema).items()
             ]
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
@@ -53,13 +67,13 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     return config
 
 
-def _comment_out_line(line):
+def comment_out_line(line):
     # If it's already is commented out (or empty), there's nothing further to do!
     stripped_line = line.lstrip()
     if not stripped_line or stripped_line.startswith('#'):
         return line
 
-    # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
+    # Comment out the names of optional options, inserting the '#' after any indent for aesthetics.
     matches = re.match(r'(\s*)', line)
     indent_spaces = matches.group(0) if matches else ''
     count_indent_spaces = len(indent_spaces)
@@ -67,7 +81,7 @@ def _comment_out_line(line):
     return '# '.join((indent_spaces, line[count_indent_spaces:]))
 
 
-def _comment_out_optional_configuration(rendered_config):
+def comment_out_optional_configuration(rendered_config):
     '''
     Post-process a rendered configuration string to comment out optional key/values, as determined
     by a sentinel in the comment before each key.
@@ -92,7 +106,7 @@ def _comment_out_optional_configuration(rendered_config):
         if not line.strip():
             optional = False
 
-        lines.append(_comment_out_line(line) if optional else line)
+        lines.append(comment_out_line(line) if optional else line)
 
     return '\n'.join(lines)
 
@@ -101,7 +115,7 @@ def render_configuration(config):
     '''
     Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
     '''
-    dumper = yaml.YAML()
+    dumper = ruamel.yaml.YAML(typ='rt')
     dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
     rendered = io.StringIO()
     dumper.dump(config, rendered)
@@ -151,7 +165,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
         return
 
     for field_name in config[0].keys():
-        field_schema = schema['items']['properties'].get(field_name, {})
+        field_schema = get_properties(schema['items']).get(field_name, {})
         description = field_schema.get('description')
 
         # No description to use? Skip it.
@@ -165,7 +179,6 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
         return
 
 
-REQUIRED_SECTION_NAMES = {'location', 'retention'}
 REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
 COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
 
@@ -179,13 +192,13 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa
         if skip_first and index == 0:
             continue
 
-        field_schema = schema['properties'].get(field_name, {})
+        field_schema = get_properties(schema).get(field_name, {})
         description = field_schema.get('description', '').strip()
 
         # If this is an optional key, add an indicator to the comment flagging it to be commented
         # out from the sample configuration. This sentinel is consumed by downstream processing that
         # does the actual commenting out.
-        if field_name not in REQUIRED_SECTION_NAMES and field_name not in REQUIRED_KEYS:
+        if field_name not in REQUIRED_KEYS:
             description = (
                 '\n'.join((description, COMMENTED_OUT_SENTINEL))
                 if description
@@ -199,7 +212,7 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa
         config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
 
         if index > 0:
-            _insert_newline_before_comment(config, field_name)
+            insert_newline_before_comment(config, field_name)
 
 
 RUAMEL_YAML_COMMENTS_INDEX = 1
@@ -226,8 +239,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
     favoring values from the source when there are collisions.
 
     The purpose of this is to upgrade configuration files from old versions of borgmatic by adding
-    new
-    configuration keys and comments.
+    new configuration keys and comments.
     '''
     if not source_config:
         return destination_config
@@ -237,7 +249,9 @@ def merge_source_configuration_into_destination(destination_config, source_confi
     for field_name, source_value in source_config.items():
         # Since this key/value is from the source configuration, leave it uncommented and remove any
         # sentinel that would cause it to get commented out.
-        remove_commented_out_sentinel(destination_config, field_name)
+        remove_commented_out_sentinel(
+            ruamel.yaml.comments.CommentedMap(destination_config), field_name
+        )
 
         # This is a mapping. Recurse for this key/value.
         if isinstance(source_value, collections.abc.Mapping):
@@ -249,7 +263,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
         # This is a sequence. Recurse for each item in it.
         if isinstance(source_value, collections.abc.Sequence) and not isinstance(source_value, str):
             destination_value = destination_config[field_name]
-            destination_config[field_name] = yaml.comments.CommentedSeq(
+            destination_config[field_name] = ruamel.yaml.comments.CommentedSeq(
                 [
                     merge_source_configuration_into_destination(
                         destination_value[index] if index < len(destination_value) else None,
@@ -267,7 +281,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
 
 
 def generate_sample_configuration(
-    source_filename, destination_filename, schema_filename, overwrite=False
+    dry_run, source_filename, destination_filename, schema_filename, overwrite=False
 ):
     '''
     Given an optional source configuration filename, and a required destination configuration
@@ -276,7 +290,7 @@ def generate_sample_configuration(
     schema. If a source filename is provided, merge the parsed contents of that configuration into
     the generated configuration.
     '''
-    schema = yaml.round_trip_load(open(schema_filename))
+    schema = ruamel.yaml.YAML(typ='safe').load(open(schema_filename))
     source_config = None
 
     if source_filename:
@@ -284,11 +298,14 @@ def generate_sample_configuration(
         normalize.normalize(source_filename, source_config)
 
     destination_config = merge_source_configuration_into_destination(
-        _schema_to_sample_configuration(schema), source_config
+        schema_to_sample_configuration(schema), source_config
     )
 
+    if dry_run:
+        return
+
     write_configuration(
         destination_filename,
-        _comment_out_optional_configuration(render_configuration(destination_config)),
+        comment_out_optional_configuration(render_configuration(destination_config)),
         overwrite=overwrite,
     )

borgmatic/config/legacy.py

@@ -1,146 +0,0 @@
-from collections import OrderedDict, namedtuple
-from configparser import RawConfigParser
-
-Section_format = namedtuple('Section_format', ('name', 'options'))
-Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
-
-
-def option(name, value_type=str, required=True):
-    '''
-    Given a config file option name, an expected type for its value, and whether it's required,
-    return a Config_option capturing that information.
-    '''
-    return Config_option(name, value_type, required)
-
-
-CONFIG_FORMAT = (
-    Section_format(
-        'location',
-        (
-            option('source_directories'),
-            option('one_file_system', value_type=bool, required=False),
-            option('remote_path', required=False),
-            option('repository'),
-        ),
-    ),
-    Section_format(
-        'storage',
-        (
-            option('encryption_passphrase', required=False),
-            option('compression', required=False),
-            option('umask', required=False),
-        ),
-    ),
-    Section_format(
-        'retention',
-        (
-            option('keep_within', required=False),
-            option('keep_hourly', int, required=False),
-            option('keep_daily', int, required=False),
-            option('keep_weekly', int, required=False),
-            option('keep_monthly', int, required=False),
-            option('keep_yearly', int, required=False),
-            option('prefix', required=False),
-        ),
-    ),
-    Section_format(
-        'consistency', (option('checks', required=False), option('check_last', required=False))
-    ),
-)
-
-
-def validate_configuration_format(parser, config_format):
-    '''
-    Given an open RawConfigParser and an expected config file format, validate that the parsed
-    configuration file has the expected sections, that any required options are present in those
-    sections, and that there aren't any unexpected options.
-
-    A section is required if any of its contained options are required.
-
-    Raise ValueError if anything is awry.
-    '''
-    section_names = set(parser.sections())
-    required_section_names = tuple(
-        section.name
-        for section in config_format
-        if any(option.required for option in section.options)
-    )
-
-    unknown_section_names = section_names - set(
-        section_format.name for section_format in config_format
-    )
-    if unknown_section_names:
-        raise ValueError(f"Unknown config sections found: {', '.join(unknown_section_names)}")
-
-    missing_section_names = set(required_section_names) - section_names
-    if missing_section_names:
-        raise ValueError(f"Missing config sections: {', '.join(missing_section_names)}")
-
-    for section_format in config_format:
-        if section_format.name not in section_names:
-            continue
-
-        option_names = parser.options(section_format.name)
-        expected_options = section_format.options
-
-        unexpected_option_names = set(option_names) - set(
-            option.name for option in expected_options
-        )
-
-        if unexpected_option_names:
-            raise ValueError(
-                f"Unexpected options found in config section {section_format.name}: {', '.join(sorted(unexpected_option_names))}",
-            )
-
-        missing_option_names = tuple(
-            option.name
-            for option in expected_options
-            if option.required
-            if option.name not in option_names
-        )
-
-        if missing_option_names:
-            raise ValueError(
-                f"Required options missing from config section {section_format.name}: {', '.join(missing_option_names)}",
-            )
-
-
-def parse_section_options(parser, section_format):
-    '''
-    Given an open RawConfigParser and an expected section format, return the option values from that
-    section as a dict mapping from option name to value. Omit those options that are not present in
-    the parsed options.
-
-    Raise ValueError if any option values cannot be coerced to the expected Python data type.
-    '''
-    type_getter = {str: parser.get, int: parser.getint, bool: parser.getboolean}
-
-    return OrderedDict(
-        (option.name, type_getter[option.value_type](section_format.name, option.name))
-        for option in section_format.options
-        if parser.has_option(section_format.name, option.name)
-    )
-
-
-def parse_configuration(config_filename, config_format):
-    '''
-    Given a config filename and an expected config file format, return the parsed configuration
-    as a namedtuple with one attribute for each parsed section.
-
-    Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
-    '''
-    parser = RawConfigParser()
-    if not parser.read(config_filename):
-        raise ValueError(f'Configuration file cannot be opened: {config_filename}')
-
-    validate_configuration_format(parser, config_format)
-
-    # Describes a parsed configuration, where each attribute is the name of a configuration file
-    # section and each value is a dict of that section's parsed options.
-    Parsed_config = namedtuple(
-        'Parsed_config', (section_format.name for section_format in config_format)
-    )
-
-    return Parsed_config(
-        *(parse_section_options(parser, section_format) for section_format in config_format)
-    )

borgmatic/config/load.py

@@ -1,6 +1,7 @@
 import functools
-import json
+import itertools
 import logging
+import operator
 import os
 
 import ruamel.yaml
@@ -8,34 +9,68 @@ import ruamel.yaml
 logger = logging.getLogger(__name__)
 
 
-def include_configuration(loader, filename_node, include_directory):
+def probe_and_include_file(filename, include_directories, config_paths):
     '''
-    Given a ruamel.yaml.loader.Loader, a ruamel.yaml.serializer.ScalarNode containing the included
-    filename, and an include directory path to search for matching files, load the given YAML
-    filename (ignoring the given loader so we can use our own) and return its contents as a data
-    structure of nested dicts and lists. If the filename is relative, probe for it within 1. the
-    current working directory and 2. the given include directory.
+    Given a filename to include, a list of include directories to search for matching files, and a
+    set of configuration paths, probe for the file, load it, and return the loaded configuration as
+    a data structure of nested dicts, lists, etc. Add the filename to the given configuration paths.
+
+    Raise FileNotFoundError if the included file was not found.
+    '''
+    expanded_filename = os.path.expanduser(filename)
+
+    if os.path.isabs(expanded_filename):
+        return load_configuration(expanded_filename, config_paths)
+
+    candidate_filenames = {
+        os.path.join(directory, expanded_filename) for directory in include_directories
+    }
+
+    for candidate_filename in candidate_filenames:
+        if os.path.exists(candidate_filename):
+            return load_configuration(candidate_filename, config_paths)
+
+    raise FileNotFoundError(
+        f'Could not find include {filename} at {" or ".join(candidate_filenames)}'
+    )
+
+
+def include_configuration(loader, filename_node, include_directory, config_paths):
+    '''
+    Given a ruamel.yaml.loader.Loader, a ruamel.yaml.nodes.ScalarNode containing the included
+    filename (or a list containing multiple such filenames), an include directory path to search for
+    matching files, and a set of configuration paths, load the given YAML filenames (ignoring the
+    given loader so we can use our own) and return their contents as data structure of nested dicts,
+    lists, etc. Add the names of included files to the given configuration paths. If the given
+    filename node's value is a scalar string, then the return value will be a single value. But if
+    the given node value is a list, then the return value will be a list of values, one per loaded
+    configuration file.
+
+    If a filename is relative, probe for it within: 1. the current working directory and 2. the
+    given include directory.
 
     Raise FileNotFoundError if an included file was not found.
     '''
     include_directories = [os.getcwd(), os.path.abspath(include_directory)]
-    include_filename = os.path.expanduser(filename_node.value)
 
-    if not os.path.isabs(include_filename):
-        candidate_filenames = [
-            os.path.join(directory, include_filename) for directory in include_directories
+    if isinstance(filename_node.value, str):
+        return probe_and_include_file(filename_node.value, include_directories, config_paths)
+
+    if (
+        isinstance(filename_node.value, list)
+        and len(filename_node.value)
+        and isinstance(filename_node.value[0], ruamel.yaml.nodes.ScalarNode)
+    ):
+        # Reversing the values ensures the correct ordering if these includes are subsequently
+        # merged together.
+        return [
+            probe_and_include_file(node.value, include_directories, config_paths)
+            for node in reversed(filename_node.value)
         ]
 
-        for candidate_filename in candidate_filenames:
-            if os.path.exists(candidate_filename):
-                include_filename = candidate_filename
-                break
-        else:
-            raise FileNotFoundError(
-                f'Could not find include {filename_node.value} at {" or ".join(candidate_filenames)}'
-            )
-
-    return load_configuration(include_filename)
+    raise ValueError(
+        'The value given for the !include tag is invalid; use a single filename or a list of filenames instead'
+    )
 
 
 def raise_retain_node_error(loader, node):
@@ -53,7 +88,7 @@ def raise_retain_node_error(loader, node):
             'The !retain tag may only be used within a configuration file containing a merged !include tag.'
         )
 
-    raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
+    raise ValueError('The !retain tag may only be used on a mapping or list.')
 
 
 def raise_omit_node_error(loader, node):
@@ -65,22 +100,31 @@ def raise_omit_node_error(loader, node):
     tags are handled by deep_merge_nodes() below.
     '''
     raise ValueError(
-        'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.'
+        'The !omit tag may only be used on a scalar (e.g., string) or list element within a configuration file containing a merged !include tag.'
     )
 
 
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
-    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    separate YAML configuration files. Example syntax: `option: !include common.yaml`
     '''
 
-    def __init__(self, preserve_quotes=None, loader=None, include_directory=None):
+    def __init__(
+        self, preserve_quotes=None, loader=None, include_directory=None, config_paths=None
+    ):
         super(Include_constructor, self).__init__(preserve_quotes, loader)
         self.add_constructor(
             '!include',
-            functools.partial(include_configuration, include_directory=include_directory),
+            functools.partial(
+                include_configuration,
+                include_directory=include_directory,
+                config_paths=config_paths,
+            ),
         )
+
+        # These are catch-all error handlers for tags that don't get applied and removed by
+        # deep_merge_nodes() below.
         self.add_constructor('!retain', raise_retain_node_error)
         self.add_constructor('!omit', raise_omit_node_error)
 
@@ -90,112 +134,152 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
         using the YAML '<<' merge key. Example syntax:
 
         ```
-        retention:
-            keep_daily: 1
+        option:
+            sub_option: 1
 
         <<: !include common.yaml
         ```
 
         These includes are deep merged into the current configuration file. For instance, in this
-        example, any "retention" options in common.yaml will get merged into the "retention" section
-        in the example configuration file.
+        example, any "option" with sub-options in common.yaml will get merged into the corresponding
+        "option" with sub-options in the example configuration file.
         '''
         representer = ruamel.yaml.representer.SafeRepresenter()
 
         for index, (key_node, value_node) in enumerate(node.value):
             if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
-                included_value = representer.represent_data(self.construct_object(value_node))
-                node.value[index] = (key_node, included_value)
+                # Replace the merge include with a sequence of included configuration nodes ready
+                # for merging. The construct_object() call here triggers include_configuration()
+                # among other constructors.
+                node.value[index] = (
+                    key_node,
+                    representer.represent_data(self.construct_object(value_node)),
+                )
 
+        # This super().flatten_mapping() call actually performs "<<" merges.
         super(Include_constructor, self).flatten_mapping(node)
 
         node.value = deep_merge_nodes(node.value)
 
 
-def load_configuration(filename):
+def load_configuration(filename, config_paths=None):
     '''
     Load the given configuration file and return its contents as a data structure of nested dicts
-    and lists. Also, replace any "{constant}" strings with the value of the "constant" key in the
-    "constants" section of the configuration file.
+    and lists. Add the filename to the given configuration paths set, and also add any included
+    configuration filenames.
 
     Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
     if there are too many recursive includes.
     '''
+    if config_paths is None:
+        config_paths = set()
 
-    # Use an embedded derived class for the include constructor so as to capture the filename
-    # value. (functools.partial doesn't work for this use case because yaml.Constructor has to be
-    # an actual class.)
-    class Include_constructor_with_include_directory(Include_constructor):
+    # Use an embedded derived class for the include constructor so as to capture the include
+    # directory and configuration paths values. (functools.partial doesn't work for this use case
+    # because yaml.Constructor has to be an actual class.)
+    class Include_constructor_with_extras(Include_constructor):
         def __init__(self, preserve_quotes=None, loader=None):
-            super(Include_constructor_with_include_directory, self).__init__(
-                preserve_quotes, loader, include_directory=os.path.dirname(filename)
+            super(Include_constructor_with_extras, self).__init__(
+                preserve_quotes,
+                loader,
+                include_directory=os.path.dirname(filename),
+                config_paths=config_paths,
             )
 
     yaml = ruamel.yaml.YAML(typ='safe')
-    yaml.Constructor = Include_constructor_with_include_directory
+    yaml.Constructor = Include_constructor_with_extras
+    config_paths.add(filename)
 
     with open(filename) as file:
-        file_contents = file.read()
-        config = yaml.load(file_contents)
-
-        if config and 'constants' in config:
-            for key, value in config['constants'].items():
-                value = json.dumps(value)
-                file_contents = file_contents.replace(f'{{{key}}}', value.strip('"'))
-
-            config = yaml.load(file_contents)
-            del config['constants']
-
-        return config
+        return yaml.load(file.read())
 
 
-def filter_omitted_nodes(nodes):
+def filter_omitted_nodes(nodes, values):
     '''
-    Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a
-    value matching such nodes.
+    Given a nested borgmatic configuration data structure as a list of tuples in the form of:
+
+    [
+        (
+            ruamel.yaml.nodes.ScalarNode as a key,
+            ruamel.yaml.nodes.MappingNode or other Node as a value,
+        ),
+        ...
+    ]
+
+    ... and a combined list of all values for those nodes, return a filtered list of the values,
+    omitting any that have an "!omit" tag (or with a value matching such nodes).
+
+    But if only a single node is given, bail and return the given values unfiltered, as "!omit" only
+    applies when there are merge includes (and therefore multiple nodes).
     '''
-    omitted_values = tuple(node.value for node in nodes if node.tag == '!omit')
+    if len(nodes) <= 1:
+        return values
 
-    return [node for node in nodes if node.value not in omitted_values]
+    omitted_values = tuple(node.value for node in values if node.tag == '!omit')
+
+    return [node for node in values if node.value not in omitted_values]
 
 
-DELETED_NODE = object()
+def merge_values(nodes):
+    '''
+    Given a nested borgmatic configuration data structure as a list of tuples in the form of:
+
+    [
+        (
+            ruamel.yaml.nodes.ScalarNode as a key,
+            ruamel.yaml.nodes.MappingNode or other Node as a value,
+        ),
+        ...
+    ]
+
+    ... merge its sequence or mapping node values and return the result. For sequence nodes, this
+    means appending together its contained lists. For mapping nodes, it means merging its contained
+    dicts.
+    '''
+    return functools.reduce(operator.add, (value.value for key, value in nodes))
 
 
 def deep_merge_nodes(nodes):
     '''
     Given a nested borgmatic configuration data structure as a list of tuples in the form of:
 
+    [
         (
             ruamel.yaml.nodes.ScalarNode as a key,
             ruamel.yaml.nodes.MappingNode or other Node as a value,
         ),
+        ...
+    ]
 
-    ... deep merge any node values corresponding to duplicate keys and return the result. If
-    there are colliding keys with non-MappingNode values (e.g., integers or strings), the last
-    of the values wins.
+    ... deep merge any node values corresponding to duplicate keys and return the result. The
+    purpose of merging like this is to support, for instance, merging one borgmatic configuration
+    file into another for reuse, such that a configuration option with sub-options does not
+    completely replace the corresponding option in a merged file.
+
+    If there are colliding keys with scalar values (e.g., integers or strings), the last of the
+    values wins.
 
     For instance, given node values of:
 
         [
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option1'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='1')
                     ),
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='7')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='2')
                     ),
                 ]),
             ),
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
                         ScalarNode(tag='tag:yaml.org,2002:int', value='5')
                     ),
                 ]),
@@ -206,81 +290,95 @@ def deep_merge_nodes(nodes):
 
         [
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option1'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='1')
                     ),
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
                         ScalarNode(tag='tag:yaml.org,2002:int', value='5')
                     ),
                 ]),
             ),
         ]
 
+    This function supports multi-way merging, meaning that if the same option name exists three or
+    more times (at the same scope level), all of those instances get merged together.
+
     If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged.
 
-    The purpose of deep merging like this is to support, for instance, merging one borgmatic
-    configuration file into another for reuse, such that a configuration section ("retention",
-    etc.) does not completely replace the corresponding section in a merged file.
+    Raise ValueError if a merge is implied using multiple incompatible types.
     '''
-    # Map from original node key/value to the replacement merged node. DELETED_NODE as a replacement
-    # node indications deletion.
-    replaced_nodes = {}
+    merged_nodes = []
 
-    # To find nodes that require merging, compare each node with each other node.
-    for a_key, a_value in nodes:
-        for b_key, b_value in nodes:
-            # If we've already considered one of the nodes for merging, skip it.
-            if (a_key, a_value) in replaced_nodes or (b_key, b_value) in replaced_nodes:
-                continue
+    def get_node_key_name(node):
+        return node[0].value
 
-            # If the keys match and the values are different, we need to merge these two A and B nodes.
-            if a_key.tag == b_key.tag and a_key.value == b_key.value and a_value != b_value:
-                # Since we're merging into the B node, consider the A node a duplicate and remove it.
-                replaced_nodes[(a_key, a_value)] = DELETED_NODE
+    # Bucket the nodes by their keys. Then merge all of the values sharing the same key.
+    for key_name, grouped_nodes in itertools.groupby(
+        sorted(nodes, key=get_node_key_name), get_node_key_name
+    ):
+        grouped_nodes = list(grouped_nodes)
 
-                # If we're dealing with MappingNodes, recurse and merge its values as well.
-                if isinstance(b_value, ruamel.yaml.nodes.MappingNode):
-                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
-                    # downstream schema validation doesn't break on our application-specific tag.
-                    if b_value.tag == '!retain':
-                        b_value.tag = 'tag:yaml.org,2002:map'
-                    else:
-                        replaced_nodes[(b_key, b_value)] = (
-                            b_key,
-                            ruamel.yaml.nodes.MappingNode(
-                                tag=b_value.tag,
-                                value=deep_merge_nodes(a_value.value + b_value.value),
-                                start_mark=b_value.start_mark,
-                                end_mark=b_value.end_mark,
-                                flow_style=b_value.flow_style,
-                                comment=b_value.comment,
-                                anchor=b_value.anchor,
-                            ),
-                        )
-                # If we're dealing with SequenceNodes, merge by appending one sequence to the other.
-                elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode):
-                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
-                    # downstream schema validation doesn't break on our application-specific tag.
-                    if b_value.tag == '!retain':
-                        b_value.tag = 'tag:yaml.org,2002:seq'
-                    else:
-                        replaced_nodes[(b_key, b_value)] = (
-                            b_key,
-                            ruamel.yaml.nodes.SequenceNode(
-                                tag=b_value.tag,
-                                value=filter_omitted_nodes(a_value.value + b_value.value),
-                                start_mark=b_value.start_mark,
-                                end_mark=b_value.end_mark,
-                                flow_style=b_value.flow_style,
-                                comment=b_value.comment,
-                                anchor=b_value.anchor,
-                            ),
-                        )
+        # The merged node inherits its attributes from the final node in the group.
+        (last_node_key, last_node_value) = grouped_nodes[-1]
+        value_types = set(type(value) for (_, value) in grouped_nodes)
 
-    return [
-        replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE
-    ]
+        if len(value_types) > 1:
+            raise ValueError(
+                f'Incompatible types found when trying to merge "{key_name}:" values across configuration files: {", ".join(value_type.id for value_type in value_types)}'
+            )
+
+        # If we're dealing with MappingNodes, recurse and merge its values as well.
+        if ruamel.yaml.nodes.MappingNode in value_types:
+            # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+            # downstream schema validation doesn't break on our application-specific tag.
+            if last_node_value.tag == '!retain' and len(grouped_nodes) > 1:
+                last_node_value.tag = 'tag:yaml.org,2002:map'
+                merged_nodes.append((last_node_key, last_node_value))
+            else:
+                merged_nodes.append(
+                    (
+                        last_node_key,
+                        ruamel.yaml.nodes.MappingNode(
+                            tag=last_node_value.tag,
+                            value=deep_merge_nodes(merge_values(grouped_nodes)),
+                            start_mark=last_node_value.start_mark,
+                            end_mark=last_node_value.end_mark,
+                            flow_style=last_node_value.flow_style,
+                            comment=last_node_value.comment,
+                            anchor=last_node_value.anchor,
+                        ),
+                    )
+                )
+
+            continue
+
+        # If we're dealing with SequenceNodes, merge by appending sequences together.
+        if ruamel.yaml.nodes.SequenceNode in value_types:
+            if last_node_value.tag == '!retain' and len(grouped_nodes) > 1:
+                last_node_value.tag = 'tag:yaml.org,2002:seq'
+                merged_nodes.append((last_node_key, last_node_value))
+            else:
+                merged_nodes.append(
+                    (
+                        last_node_key,
+                        ruamel.yaml.nodes.SequenceNode(
+                            tag=last_node_value.tag,
+                            value=filter_omitted_nodes(grouped_nodes, merge_values(grouped_nodes)),
+                            start_mark=last_node_value.start_mark,
+                            end_mark=last_node_value.end_mark,
+                            flow_style=last_node_value.flow_style,
+                            comment=last_node_value.comment,
+                            anchor=last_node_value.anchor,
+                        ),
+                    )
+                )
+
+            continue
+
+        merged_nodes.append((last_node_key, last_node_value))
+
+    return merged_nodes

borgmatic/config/normalize.py

@@ -2,67 +2,244 @@ import logging
 import os
 
 
+def normalize_sections(config_filename, config):
+    '''
+    Given a configuration filename and a configuration dict of its loaded contents, airlift any
+    options out of sections ("location:", etc.) to the global scope and delete those sections.
+    Return any log message warnings produced based on the normalization performed.
+
+    Raise ValueError if the "prefix" option is set in both "location" and "consistency" sections.
+    '''
+    try:
+        location = config.get('location') or {}
+    except AttributeError:
+        raise ValueError('Configuration does not contain any options')
+
+    storage = config.get('storage') or {}
+    consistency = config.get('consistency') or {}
+    hooks = config.get('hooks') or {}
+
+    if (
+        location.get('prefix')
+        and consistency.get('prefix')
+        and location.get('prefix') != consistency.get('prefix')
+    ):
+        raise ValueError(
+            'The retention prefix and the consistency prefix cannot have different values (unless one is not set).'
+        )
+
+    if storage.get('umask') and hooks.get('umask') and storage.get('umask') != hooks.get('umask'):
+        raise ValueError(
+            'The storage umask and the hooks umask cannot have different values (unless one is not set).'
+        )
+
+    any_section_upgraded = False
+
+    # Move any options from deprecated sections into the global scope.
+    for section_name in ('location', 'storage', 'retention', 'consistency', 'output', 'hooks'):
+        section_config = config.get(section_name)
+
+        if section_config is not None:
+            any_section_upgraded = True
+            del config[section_name]
+            config.update(section_config)
+
+    if any_section_upgraded:
+        return [
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: Configuration sections (like location:, storage:, retention:, consistency:, and hooks:) are deprecated and support will be removed from a future release. To prepare for this, move your options out of sections to the global scope.',
+                )
+            )
+        ]
+
+    return []
+
+
 def normalize(config_filename, config):
     '''
     Given a configuration filename and a configuration dict of its loaded contents, apply particular
     hard-coded rules to normalize the configuration to adhere to the current schema. Return any log
     message warnings produced based on the normalization performed.
+
+    Raise ValueError the configuration cannot be normalized.
     '''
-    logs = []
-    location = config.get('location') or {}
-    storage = config.get('storage') or {}
-    consistency = config.get('consistency') or {}
-    hooks = config.get('hooks') or {}
+    logs = normalize_sections(config_filename, config)
+
+    if config.get('borgmatic_source_directory'):
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The borgmatic_source_directory option is deprecated and will be removed from a future release. Use borgmatic_runtime_directory and borgmatic_state_directory instead.',
+                )
+            )
+        )
 
     # Upgrade exclude_if_present from a string to a list.
-    exclude_if_present = location.get('exclude_if_present')
+    exclude_if_present = config.get('exclude_if_present')
     if isinstance(exclude_if_present, str):
-        config['location']['exclude_if_present'] = [exclude_if_present]
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The exclude_if_present option now expects a list value. String values for this option are deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['exclude_if_present'] = [exclude_if_present]
+
+    # Unconditionally set the bootstrap hook so that it's enabled by default and config files get
+    # stored in each Borg archive.
+    config.setdefault('bootstrap', {})
+
+    # Move store_config_files from the global scope to the bootstrap hook.
+    store_config_files = config.get('store_config_files')
+    if store_config_files is not None:
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The store_config_files option has moved under the bootstrap hook. Specifying store_config_files at the global scope is deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        del config['store_config_files']
+        config['bootstrap']['store_config_files'] = store_config_files
 
     # Upgrade various monitoring hooks from a string to a dict.
-    healthchecks = hooks.get('healthchecks')
+    healthchecks = config.get('healthchecks')
     if isinstance(healthchecks, str):
-        config['hooks']['healthchecks'] = {'ping_url': healthchecks}
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The healthchecks hook now expects a key/value pair with "ping_url" as a key. String values for this option are deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['healthchecks'] = {'ping_url': healthchecks}
 
-    cronitor = hooks.get('cronitor')
+    cronitor = config.get('cronitor')
     if isinstance(cronitor, str):
-        config['hooks']['cronitor'] = {'ping_url': cronitor}
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['cronitor'] = {'ping_url': cronitor}
 
-    pagerduty = hooks.get('pagerduty')
+    pagerduty = config.get('pagerduty')
     if isinstance(pagerduty, str):
-        config['hooks']['pagerduty'] = {'integration_key': pagerduty}
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['pagerduty'] = {'integration_key': pagerduty}
 
-    cronhub = hooks.get('cronhub')
+    cronhub = config.get('cronhub')
     if isinstance(cronhub, str):
-        config['hooks']['cronhub'] = {'ping_url': cronhub}
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['cronhub'] = {'ping_url': cronhub}
 
     # Upgrade consistency checks from a list of strings to a list of dicts.
-    checks = consistency.get('checks')
+    checks = config.get('checks')
     if isinstance(checks, list) and len(checks) and isinstance(checks[0], str):
-        config['consistency']['checks'] = [{'name': check_type} for check_type in checks]
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The checks option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['checks'] = [{'name': check_type} for check_type in checks]
 
     # Rename various configuration options.
-    numeric_owner = location.pop('numeric_owner', None)
+    numeric_owner = config.pop('numeric_owner', None)
     if numeric_owner is not None:
-        config['location']['numeric_ids'] = numeric_owner
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The numeric_owner option has been renamed to numeric_ids. numeric_owner is deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['numeric_ids'] = numeric_owner
 
-    bsd_flags = location.pop('bsd_flags', None)
+    bsd_flags = config.pop('bsd_flags', None)
     if bsd_flags is not None:
-        config['location']['flags'] = bsd_flags
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The bsd_flags option has been renamed to flags. bsd_flags is deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['flags'] = bsd_flags
 
-    remote_rate_limit = storage.pop('remote_rate_limit', None)
+    remote_rate_limit = config.pop('remote_rate_limit', None)
     if remote_rate_limit is not None:
-        config['storage']['upload_rate_limit'] = remote_rate_limit
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The remote_rate_limit option has been renamed to upload_rate_limit. remote_rate_limit is deprecated and support will be removed from a future release.',
+                )
+            )
+        )
+        config['upload_rate_limit'] = remote_rate_limit
 
     # Upgrade remote repositories to ssh:// syntax, required in Borg 2.
-    repositories = location.get('repositories')
+    repositories = config.get('repositories')
     if repositories:
-        if isinstance(repositories[0], str):
-            config['location']['repositories'] = [
-                {'path': repository} for repository in repositories
+        if any(isinstance(repository, str) for repository in repositories):
+            logs.append(
+                logging.makeLogRecord(
+                    dict(
+                        levelno=logging.WARNING,
+                        levelname='WARNING',
+                        msg=f'{config_filename}: The repositories option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.',
+                    )
+                )
+            )
+            config['repositories'] = [
+                {'path': repository} if isinstance(repository, str) else repository
+                for repository in repositories
             ]
-            repositories = config['location']['repositories']
-        config['location']['repositories'] = []
+            repositories = config['repositories']
+
+        config['repositories'] = []
+
         for repository_dict in repositories:
             repository_path = repository_dict['path']
             if '~' in repository_path:
@@ -71,7 +248,7 @@ def normalize(config_filename, config):
                         dict(
                             levelno=logging.WARNING,
                             levelname='WARNING',
-                            msg=f'{config_filename}: Repository paths containing "~" are deprecated in borgmatic and no longer work in Borg 2.x+.',
+                            msg=f'{config_filename}: Repository paths containing "~" are deprecated in borgmatic and support will be removed from a future release.',
                         )
                     )
                 )
@@ -80,14 +257,18 @@ def normalize(config_filename, config):
                     updated_repository_path = os.path.abspath(
                         repository_path.partition('file://')[-1]
                     )
-                    config['location']['repositories'].append(
+                    config['repositories'].append(
                         dict(
                             repository_dict,
                             path=updated_repository_path,
                         )
                     )
-                elif repository_path.startswith('ssh://'):
-                    config['location']['repositories'].append(repository_dict)
+                elif (
+                    repository_path.startswith('ssh://')
+                    or repository_path.startswith('sftp://')
+                    or repository_path.startswith('rclone:')
+                ):
+                    config['repositories'].append(repository_dict)
                 else:
                     rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
                     logs.append(
@@ -95,17 +276,28 @@ def normalize(config_filename, config):
                             dict(
                                 levelno=logging.WARNING,
                                 levelname='WARNING',
-                                msg=f'{config_filename}: Remote repository paths without ssh:// syntax are deprecated. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
+                                msg=f'{config_filename}: Remote repository paths without ssh:// or rclone: syntax are deprecated and support will be removed from a future release. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
                             )
                         )
                     )
-                    config['location']['repositories'].append(
+                    config['repositories'].append(
                         dict(
                             repository_dict,
                             path=rewritten_repository_path,
                         )
                     )
             else:
-                config['location']['repositories'].append(repository_dict)
+                config['repositories'].append(repository_dict)
+
+    if config.get('prefix'):
+        logs.append(
+            logging.makeLogRecord(
+                dict(
+                    levelno=logging.WARNING,
+                    levelname='WARNING',
+                    msg=f'{config_filename}: The prefix option is deprecated and support will be removed from a future release. Use archive_name_format or match_archives instead.',
+                )
+            )
+        )
 
     return logs

borgmatic/config/override.py

@@ -13,6 +13,11 @@ def set_values(config, keys, value):
 
     first_key = keys[0]
     if len(keys) == 1:
+        if isinstance(config, list):
+            raise ValueError(
+                'When overriding a list option, the value must use list syntax (e.g., "[foo, bar]" or "[{key: value}]" as appropriate)'
+            )
+
         config[first_key] = value
         return
 
@@ -22,29 +27,70 @@ def set_values(config, keys, value):
     set_values(config[first_key], keys[1:], value)
 
 
-def convert_value_type(value):
+def convert_value_type(value, option_type):
     '''
-    Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
-    converted to that type.
+    Given a string value and its schema type as a string, determine its logical type (string,
+    boolean, integer, etc.), and return it converted to that type.
+
+    If the option type is a string, leave the value as a string so that special characters in it
+    don't get interpreted as YAML during conversion.
 
     Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML.
     '''
+    if option_type == 'string':
+        return value
+
     return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
 
 
-def parse_overrides(raw_overrides):
-    '''
-    Given a sequence of configuration file override strings in the form of "section.option=value",
-    parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
-    instance, given the following raw overrides:
+LEGACY_SECTION_NAMES = {'location', 'storage', 'retention', 'consistency', 'output', 'hooks'}
 
-        ['section.my_option=value1', 'section.other_option=value2']
+
+def strip_section_names(parsed_override_key):
+    '''
+    Given a parsed override key as a tuple of option and suboption names, strip out any initial
+    legacy section names, since configuration file normalization also strips them out.
+    '''
+    if parsed_override_key[0] in LEGACY_SECTION_NAMES:
+        return parsed_override_key[1:]
+
+    return parsed_override_key
+
+
+def type_for_option(schema, option_keys):
+    '''
+    Given a configuration schema and a sequence of keys identifying an option, e.g.
+    ('extra_borg_options', 'init'), return the schema type of that option as a string.
+
+    Return None if the option or its type cannot be found in the schema.
+    '''
+    option_schema = schema
+
+    for key in option_keys:
+        try:
+            option_schema = option_schema['properties'][key]
+        except KeyError:
+            return None
+
+    try:
+        return option_schema['type']
+    except KeyError:
+        return None
+
+
+def parse_overrides(raw_overrides, schema):
+    '''
+    Given a sequence of configuration file override strings in the form of "option.suboption=value"
+    and a configuration schema dict, parse and return a sequence of tuples (keys, values), where
+    keys is a sequence of strings. For instance, given the following raw overrides:
+
+        ['my_option.suboption=value1', 'other_option=value2']
 
     ... return this:
 
         (
-            (('section', 'my_option'), 'value1'),
-            (('section', 'other_option'), 'value2'),
+            (('my_option', 'suboption'), 'value1'),
+            (('other_option'), 'value2'),
         )
 
     Raise ValueError if an override can't be parsed.
@@ -57,15 +103,18 @@ def parse_overrides(raw_overrides):
     for raw_override in raw_overrides:
         try:
             raw_keys, value = raw_override.split('=', 1)
+            keys = tuple(raw_keys.split('.'))
+            option_type = type_for_option(schema, keys)
+
             parsed_overrides.append(
                 (
-                    tuple(raw_keys.split('.')),
-                    convert_value_type(value),
+                    keys,
+                    convert_value_type(value, option_type),
                 )
             )
         except ValueError:
             raise ValueError(
-                f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE"
+                f"Invalid override '{raw_override}'. Make sure you use the form: OPTION=VALUE or OPTION.SUBOPTION=VALUE"
             )
         except ruamel.yaml.error.YAMLError as error:
             raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
@@ -73,12 +122,18 @@ def parse_overrides(raw_overrides):
     return tuple(parsed_overrides)
 
 
-def apply_overrides(config, raw_overrides):
+def apply_overrides(config, schema, raw_overrides):
     '''
-    Given a configuration dict and a sequence of configuration file override strings in the form of
-    "section.option=value", parse each override and set it the configuration dict.
+    Given a configuration dict, a corresponding configuration schema dict, and a sequence of
+    configuration file override strings in the form of "option.suboption=value", parse each override
+    and set it into the configuration dict.
+
+    Set the overrides into the configuration both with and without deprecated section names (if
+    used), so that the overrides work regardless of whether the configuration is also using
+    deprecated section names.
     '''
-    overrides = parse_overrides(raw_overrides)
+    overrides = parse_overrides(raw_overrides, schema)
 
     for keys, value in overrides:
         set_values(config, keys, value)
+        set_values(config, strip_section_names(keys), value)

borgmatic/config/paths.py

@@ -0,0 +1,179 @@
+import logging
+import os
+import tempfile
+
+logger = logging.getLogger(__name__)
+
+
+def expand_user_in_path(path):
+    '''
+    Given a directory path, expand any tildes in it.
+    '''
+    try:
+        return os.path.expanduser(path or '') or None
+    except TypeError:
+        return None
+
+
+def get_working_directory(config):  # pragma: no cover
+    '''
+    Given a configuration dict, get the working directory from it, expanding any tildes.
+    '''
+    return expand_user_in_path(config.get('working_directory'))
+
+
+def get_borgmatic_source_directory(config):
+    '''
+    Given a configuration dict, get the (deprecated) borgmatic source directory, expanding any
+    tildes. Defaults to ~/.borgmatic.
+    '''
+    return expand_user_in_path(config.get('borgmatic_source_directory') or '~/.borgmatic')
+
+
+TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-'
+
+
+def replace_temporary_subdirectory_with_glob(
+    path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX
+):
+    '''
+    Given an absolute temporary directory path and an optional temporary directory prefix, look for
+    a subdirectory within it starting with the temporary directory prefix (or a default) and replace
+    it with an appropriate glob. For instance, given:
+
+        /tmp/borgmatic-aet8kn93/borgmatic
+
+    ... replace it with:
+
+        /tmp/borgmatic-*/borgmatic
+
+    This is useful for finding previous temporary directories from prior borgmatic runs.
+    '''
+    return os.path.join(
+        '/',
+        *(
+            (
+                f'{temporary_directory_prefix}*'
+                if subdirectory.startswith(temporary_directory_prefix)
+                else subdirectory
+            )
+            for subdirectory in path.split(os.path.sep)
+        ),
+    )
+
+
+class Runtime_directory:
+    '''
+    A Python context manager for creating and cleaning up the borgmatic runtime directory used for
+    storing temporary runtime data like streaming database dumps and bootstrap metadata.
+
+    Example use as a context manager:
+
+        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+            do_something_with(borgmatic_runtime_directory)
+
+    For the scope of that "with" statement, the runtime directory is available. Afterwards, it
+    automatically gets cleaned up as necessary.
+    '''
+
+    def __init__(self, config):
+        '''
+        Given a configuration dict determine the borgmatic runtime directory, creating a secure,
+        temporary directory within it if necessary. Defaults to $XDG_RUNTIME_DIR/./borgmatic or
+        $RUNTIME_DIRECTORY/./borgmatic or $TMPDIR/borgmatic-[random]/./borgmatic or
+        $TEMP/borgmatic-[random]/./borgmatic or /tmp/borgmatic-[random]/./borgmatic where "[random]"
+        is a randomly generated string intended to avoid path collisions.
+
+        If XDG_RUNTIME_DIR or RUNTIME_DIRECTORY is set and already ends in "/borgmatic", then don't
+        tack on a second "/borgmatic" path component.
+
+        The "/./" is taking advantage of a Borg feature such that the part of the path before the "/./"
+        does not get stored in the file path within an archive. That way, the path of the runtime
+        directory can change without leaving database dumps within an archive inaccessible.
+        '''
+        runtime_directory = (
+            config.get('user_runtime_directory')
+            or os.environ.get('XDG_RUNTIME_DIR')  # Set by PAM on Linux.
+            or os.environ.get('RUNTIME_DIRECTORY')  # Set by systemd if configured.
+        )
+
+        if runtime_directory:
+            if not runtime_directory.startswith(os.path.sep):
+                raise ValueError('The runtime directory must be an absolute path')
+
+            self.temporary_directory = None
+        else:
+            base_directory = os.environ.get('TMPDIR') or os.environ.get('TEMP') or '/tmp'
+
+            if not base_directory.startswith(os.path.sep):
+                raise ValueError('The temporary directory must be an absolute path')
+
+            os.makedirs(base_directory, mode=0o700, exist_ok=True)
+            self.temporary_directory = tempfile.TemporaryDirectory(
+                prefix=TEMPORARY_DIRECTORY_PREFIX,
+                dir=base_directory,
+            )
+            runtime_directory = self.temporary_directory.name
+
+        (base_path, final_directory) = os.path.split(runtime_directory.rstrip(os.path.sep))
+
+        self.runtime_path = expand_user_in_path(
+            os.path.join(
+                base_path if final_directory == 'borgmatic' else runtime_directory,
+                '.',  # Borg 1.4+ "slashdot" hack.
+                'borgmatic',
+            )
+        )
+        os.makedirs(self.runtime_path, mode=0o700, exist_ok=True)
+
+        logger.debug(f'Using runtime directory {os.path.normpath(self.runtime_path)}')
+
+    def __enter__(self):
+        '''
+        Return the borgmatic runtime path as a string.
+        '''
+        return self.runtime_path
+
+    def __exit__(self, exception, value, traceback):
+        '''
+        Delete any temporary directory that was created as part of initialization.
+        '''
+        if self.temporary_directory:
+            try:
+                self.temporary_directory.cleanup()
+            # The cleanup() call errors if, for instance, there's still a
+            # mounted filesystem within the temporary directory. There's
+            # nothing we can do about that here, so swallow the error.
+            except OSError:
+                pass
+
+
+def make_runtime_directory_glob(borgmatic_runtime_directory):
+    '''
+    Given a borgmatic runtime directory path, make a glob that would match that path, specifically
+    replacing any randomly generated temporary subdirectory with "*" since such a directory's name
+    changes on every borgmatic run.
+    '''
+    return os.path.join(
+        *(
+            '*' if subdirectory.startswith(TEMPORARY_DIRECTORY_PREFIX) else subdirectory
+            for subdirectory in os.path.normpath(borgmatic_runtime_directory).split(os.path.sep)
+        )
+    )
+
+
+def get_borgmatic_state_directory(config):
+    '''
+    Given a configuration dict, get the borgmatic state directory used for storing borgmatic state
+    files like records of when checks last ran. Defaults to $XDG_STATE_HOME/borgmatic or
+    ~/.local/state/./borgmatic.
+    '''
+    return expand_user_in_path(
+        os.path.join(
+            config.get('user_state_directory')
+            or os.environ.get('XDG_STATE_HOME')
+            or os.environ.get('STATE_DIRECTORY')  # Set by systemd if configured.
+            or '~/.local/state',
+            'borgmatic',
+        )
+    )

borgmatic/config/validate.py

@@ -1,10 +1,11 @@
+import fnmatch
 import os
 
 import jsonschema
 import ruamel.yaml
 
 import borgmatic.config
-from borgmatic.config import environment, load, normalize, override
+from borgmatic.config import constants, environment, load, normalize, override
 
 
 def schema_filename():
@@ -71,18 +72,15 @@ def apply_logical_validation(config_filename, parsed_configuration):
     below), run through any additional logical validation checks. If there are any such validation
     problems, raise a Validation_error.
     '''
-    location_repositories = parsed_configuration.get('location', {}).get('repositories')
-    check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
+    repositories = parsed_configuration.get('repositories')
+    check_repositories = parsed_configuration.get('check_repositories', [])
     for repository in check_repositories:
         if not any(
-            repositories_match(repository, config_repository)
-            for config_repository in location_repositories
+            repositories_match(repository, config_repository) for config_repository in repositories
         ):
             raise Validation_error(
                 config_filename,
-                (
-                    f'Unknown repository in the "consistency" section\'s "check_repositories": {repository}',
-                ),
+                (f'Unknown repository in "check_repositories": {repository}',),
             )
 
 
@@ -90,33 +88,44 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form
-    of "section.option=value", return the parsed configuration as a data structure of nested dicts
-    and lists corresponding to the schema. Example return value:
+    of "option.suboption=value", and whether to resolve environment variables, return the parsed
+    configuration as a data structure of nested dicts and lists corresponding to the schema. Example
+    return value:
 
-       {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
-       'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
+        {
+            'source_directories': ['/home', '/etc'],
+            'repository': 'hostname.borg',
+            'keep_daily': 7,
+            'checks': ['repository', 'archives'],
+        }
 
-    Also return a sequence of logging.LogRecord instances containing any warnings about the
-    configuration.
+    Also return a set of loaded configuration paths and a sequence of logging.LogRecord instances
+    containing any warnings about the configuration.
 
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
+    config_paths = set()
+
     try:
-        config = load.load_configuration(config_filename)
+        config = load.load_configuration(config_filename, config_paths)
         schema = load.load_configuration(schema_filename)
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    override.apply_overrides(config, overrides)
-    logs = normalize.normalize(config_filename, config)
+    override.apply_overrides(config, schema, overrides)
+    constants.apply_constants(config, config.get('constants') if config else {})
+
     if resolve_env:
         environment.resolve_env_variables(config)
 
+    logs = normalize.normalize(config_filename, config)
+
     try:
         validator = jsonschema.Draft7Validator(schema)
     except AttributeError:  # pragma: no cover
         validator = jsonschema.Draft4Validator(schema)
+
     validation_errors = tuple(validator.iter_errors(config))
 
     if validation_errors:
@@ -126,7 +135,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
 
     apply_logical_validation(config_filename, config)
 
-    return config, logs
+    return config, config_paths, logs
 
 
 def normalize_repository_path(repository):
@@ -143,29 +152,40 @@ def normalize_repository_path(repository):
         return repository
 
 
+def glob_match(first, second):
+    '''
+    Given two strings, return whether the first matches the second. Globs are
+    supported.
+    '''
+    if first is None or second is None:
+        return False
+
+    return fnmatch.fnmatch(first, second) or fnmatch.fnmatch(second, first)
+
+
 def repositories_match(first, second):
     '''
-    Given two repository dicts with keys 'path' (relative and/or absolute),
-    and 'label', or two repository paths, return whether they match.
+    Given two repository dicts with keys "path" (relative and/or absolute),
+    and "label", two repository paths as strings, or a mix of the two formats,
+    return whether they match. Globs are supported.
     '''
     if isinstance(first, str):
         first = {'path': first, 'label': first}
     if isinstance(second, str):
         second = {'path': second, 'label': second}
-    return (first.get('label') == second.get('label')) or (
-        normalize_repository_path(first.get('path'))
-        == normalize_repository_path(second.get('path'))
+
+    return glob_match(first.get('label'), second.get('label')) or glob_match(
+        normalize_repository_path(first.get('path')), normalize_repository_path(second.get('path'))
     )
 
 
 def guard_configuration_contains_repository(repository, configurations):
     '''
     Given a repository path and a dict mapping from config filename to corresponding parsed config
-    dict, ensure that the repository is declared exactly once in all of the configurations. If no
+    dict, ensure that the repository is declared at least once in all of the configurations. If no
     repository is given, skip this check.
 
-    Raise ValueError if the repository is not found in a configuration, or is declared multiple
-    times.
+    Raise ValueError if the repository is not found in any configurations.
     '''
     if not repository:
         return
@@ -174,35 +194,10 @@ def guard_configuration_contains_repository(repository, configurations):
         tuple(
             config_repository
             for config in configurations.values()
-            for config_repository in config['location']['repositories']
+            for config_repository in config['repositories']
             if repositories_match(config_repository, repository)
         )
     )
 
     if count == 0:
-        raise ValueError(f'Repository {repository} not found in configuration files')
-    if count > 1:
-        raise ValueError(f'Repository {repository} found in multiple configuration files')
-
-
-def guard_single_repository_selected(repository, configurations):
-    '''
-    Given a repository path and a dict mapping from config filename to corresponding parsed config
-    dict, ensure either a single repository exists across all configuration files or a repository
-    path was given.
-    '''
-    if repository:
-        return
-
-    count = len(
-        tuple(
-            config_repository
-            for config in configurations.values()
-            for config_repository in config['location']['repositories']
-        )
-    )
-
-    if count != 1:
-        raise ValueError(
-            "Can't determine which repository to use. Use --repository to disambiguate"
-        )
+        raise ValueError(f'Repository "{repository}" not found in configuration files')

borgmatic/execute.py

@@ -1,29 +1,71 @@
 import collections
+import enum
 import logging
-import os
 import select
 import subprocess
+import textwrap
+
+import borgmatic.logger
 
 logger = logging.getLogger(__name__)
 
 
 ERROR_OUTPUT_MAX_LINE_COUNT = 25
-BORG_ERROR_EXIT_CODE = 2
+BORG_ERROR_EXIT_CODE_START = 2
+BORG_ERROR_EXIT_CODE_END = 99
 
 
-def exit_code_indicates_error(command, exit_code, borg_local_path=None):
+class Exit_status(enum.Enum):
+    STILL_RUNNING = 1
+    SUCCESS = 2
+    WARNING = 3
+    ERROR = 4
+
+
+def interpret_exit_code(command, exit_code, borg_local_path=None, borg_exit_codes=None):
     '''
-    Return True if the given exit code from running a command corresponds to an error. If a Borg
-    local path is given and matches the process' command, then treat exit code 1 as a warning
-    instead of an error.
+    Return an Exit_status value (e.g. SUCCESS, ERROR, or WARNING) based on interpreting the given
+    exit code. If a Borg local path is given and matches the process' command, then interpret the
+    exit code based on Borg's documented exit code semantics. And if Borg exit codes are given as a
+    sequence of exit code configuration dicts, then take those configured preferences into account.
     '''
     if exit_code is None:
-        return False
+        return Exit_status.STILL_RUNNING
+    if exit_code == 0:
+        return Exit_status.SUCCESS
 
     if borg_local_path and command[0] == borg_local_path:
-        return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE)
+        # First try looking for the exit code in the borg_exit_codes configuration.
+        for entry in borg_exit_codes or ():
+            if entry.get('code') == exit_code:
+                treat_as = entry.get('treat_as')
 
-    return bool(exit_code != 0)
+                if treat_as == 'error':
+                    logger.error(
+                        f'Treating exit code {exit_code} as an error, as per configuration'
+                    )
+                    return Exit_status.ERROR
+                elif treat_as == 'warning':
+                    logger.warning(
+                        f'Treating exit code {exit_code} as a warning, as per configuration'
+                    )
+                    return Exit_status.WARNING
+
+        # If the exit code doesn't have explicit configuration, then fall back to the default Borg
+        # behavior.
+        return (
+            Exit_status.ERROR
+            if (
+                exit_code < 0
+                or (
+                    exit_code >= BORG_ERROR_EXIT_CODE_START
+                    and exit_code <= BORG_ERROR_EXIT_CODE_END
+                )
+            )
+            else Exit_status.WARNING
+        )
+
+    return Exit_status.ERROR
 
 
 def command_for_process(process):
@@ -60,7 +102,7 @@ def append_last_lines(last_lines, captured_output, line, output_log_level):
         logger.log(output_log_level, line)
 
 
-def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
+def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes):
     '''
     Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
     process with the requested log level. Additionally, raise a CalledProcessError if a process
@@ -68,7 +110,8 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
     path).
 
     If output log level is None, then instead of logging, capture output for each process and return
-    it as a dict from the process to its output.
+    it as a dict from the process to its output. Use the given Borg local path and exit code
+    configuration to decide what's an error and what's a warning.
 
     For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
     stdouts are given to exclude, then for any matching processes, log from their stderr instead.
@@ -132,10 +175,13 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
 
             if exit_code is None:
                 still_running = True
+                command = process.args.split(' ') if isinstance(process.args, str) else process.args
+                continue
 
             command = process.args.split(' ') if isinstance(process.args, str) else process.args
-            # If any process errors, then raise accordingly.
-            if exit_code_indicates_error(command, exit_code, borg_local_path):
+            exit_status = interpret_exit_code(command, exit_code, borg_local_path, borg_exit_codes)
+
+            if exit_status in (Exit_status.ERROR, Exit_status.WARNING):
                 # If an error occurs, include its output in the raised exception so that we don't
                 # inadvertently hide error output.
                 output_buffer = output_buffer_for_process(process, exclude_stdouts)
@@ -161,9 +207,13 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
                         other_process.stdout.read(0)
                         other_process.kill()
 
-                raise subprocess.CalledProcessError(
-                    exit_code, command_for_process(process), '\n'.join(last_lines)
-                )
+                if exit_status == Exit_status.ERROR:
+                    raise subprocess.CalledProcessError(
+                        exit_code, command_for_process(process), '\n'.join(last_lines)
+                    )
+
+                still_running = False
+                break
 
     if captured_outputs:
         return {
@@ -171,19 +221,57 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         }
 
 
-def log_command(full_command, input_file=None, output_file=None):
+SECRET_COMMAND_FLAG_NAMES = {'--password'}
+
+
+def mask_command_secrets(full_command):
+    '''
+    Given a command as a sequence, mask secret values for flags like "--password" in preparation for
+    logging.
+    '''
+    masked_command = []
+    previous_piece = None
+
+    for piece in full_command:
+        masked_command.append('***' if previous_piece in SECRET_COMMAND_FLAG_NAMES else piece)
+        previous_piece = piece
+
+    return tuple(masked_command)
+
+
+MAX_LOGGED_COMMAND_LENGTH = 1000
+
+
+PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG = ('BORG_', 'PG', 'MARIADB_', 'MYSQL_')
+
+
+def log_command(full_command, input_file=None, output_file=None, environment=None):
     '''
     Log the given command (a sequence of command/argument strings), along with its input/output file
-    paths.
+    paths and extra environment variables (with omitted values in case they contain passwords).
     '''
     logger.debug(
-        ' '.join(full_command)
-        + (f" < {getattr(input_file, 'name', '')}" if input_file else '')
-        + (f" > {getattr(output_file, 'name', '')}" if output_file else '')
+        textwrap.shorten(
+            ' '.join(
+                tuple(
+                    f'{key}=***'
+                    for key in (environment or {}).keys()
+                    if any(
+                        key.startswith(prefix)
+                        for prefix in PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG
+                    )
+                )
+                + mask_command_secrets(full_command)
+            ),
+            width=MAX_LOGGED_COMMAND_LENGTH,
+            placeholder=' ...',
+        )
+        + (f" < {getattr(input_file, 'name', input_file)}" if input_file else '')
+        + (f" > {getattr(output_file, 'name', output_file)}" if output_file else '')
     )
 
 
-# An sentinel passed as an output file to execute_command() to indicate that the command's output
+# A sentinel passed as an output file to execute_command() to indicate that the command's output
 # should be allowed to flow through to stdout without being captured for logging. Useful for
 # commands with interactive prompts or those that mess directly with the console.
 DO_NOT_CAPTURE = object()
@@ -195,26 +283,27 @@ def execute_command(
     output_file=None,
     input_file=None,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
+    borg_exit_codes=None,
     run_to_completion=True,
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
     given log level. If an open output file object is given, then write stdout to the file and only
     log stderr. If an open input file object is given, then read stdin from the file. If shell is
-    True, execute the command within a shell. If an extra environment dict is given, then use it to
-    augment the current environment, and pass the result into the command. If a working directory is
-    given, use that as the present working directory when running the command. If a Borg local path
-    is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning
-    instead of an error. If run to completion is False, then return the process for the command
-    without executing it to completion.
+    True, execute the command within a shell. If an environment variables dict is given, then pass
+    it into the command. If a working directory is given, use that as the present working directory
+    when running the command. If a Borg local path is given, and the command matches it (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are
+    given as a sequence of exit code configuration dicts, then use that configuration to decide
+    what's an error and what's a warning. If run to completion is False, then return the process for
+    the command without executing it to completion.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    log_command(full_command, input_file, output_file)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, output_file, environment)
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
@@ -226,45 +315,64 @@ def execute_command(
         shell=shell,
         env=environment,
         cwd=working_directory,
+        # Necessary for passing credentials via anonymous pipe.
+        close_fds=False,
     )
     if not run_to_completion:
         return process
 
-    log_outputs(
-        (process,), (input_file, output_file), output_log_level, borg_local_path=borg_local_path
-    )
+    with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
+        log_outputs(
+            (process,),
+            (input_file, output_file),
+            output_log_level,
+            borg_local_path,
+            borg_exit_codes,
+        )
 
 
 def execute_command_and_capture_output(
     full_command,
+    input_file=None,
     capture_stderr=False,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
+    borg_local_path=None,
+    borg_exit_codes=None,
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its
-    output (stdout). If capture stderr is True, then capture and return stderr in addition to
-    stdout. If shell is True, execute the command within a shell. If an extra environment dict is
-    given, then use it to augment the current environment, and pass the result into the command. If
-    a working directory is given, use that as the present working directory when running the command.
+    output (stdout). If an input file descriptor is given, then pipe it to the command's stdin. If
+    capture stderr is True, then capture and return stderr in addition to stdout. If shell is True,
+    execute the command within a shell. If an environment variables dict is given, then pass it into
+    the command. If a working directory is given, use that as the present working directory when
+    running the command. If a Borg local path is given, and the command matches it (regardless of
+    arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are given
+    as a sequence of exit code configuration dicts, then use that configuration to decide what's an
+    error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    log_command(full_command)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, environment=environment)
     command = ' '.join(full_command) if shell else full_command
 
     try:
         output = subprocess.check_output(
             command,
+            stdin=input_file,
             stderr=subprocess.STDOUT if capture_stderr else None,
             shell=shell,
             env=environment,
             cwd=working_directory,
+            # Necessary for passing credentials via anonymous pipe.
+            close_fds=False,
         )
     except subprocess.CalledProcessError as error:
-        if exit_code_indicates_error(command, error.returncode):
+        if (
+            interpret_exit_code(command, error.returncode, borg_local_path, borg_exit_codes)
+            == Exit_status.ERROR
+        ):
             raise
         output = error.output
 
@@ -278,9 +386,10 @@ def execute_command_with_processes(
     output_file=None,
     input_file=None,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
+    borg_exit_codes=None,
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
@@ -291,17 +400,17 @@ def execute_command_with_processes(
     If an open output file object is given, then write stdout to the file and only log stderr. But
     if output log level is None, instead suppress logging and return the captured output for (only)
     the given command. If an open input file object is given, then read stdin from the file. If
-    shell is True, execute the command within a shell. If an extra environment dict is given, then
-    use it to augment the current environment, and pass the result into the command. If a working
-    directory is given, use that as the present working directory when running the command. If a
-    Borg local path is given, then for any matching command or process (regardless of arguments),
-    treat exit code 1 as a warning instead of an error.
+    shell is True, execute the command within a shell. If an environment variables dict is given,
+    then pass it into the command. If a working directory is given, use that as the present working
+    directory when running the command. If a Borg local path is given, then for any matching command
+    or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if
+    Borg exit codes are given as a sequence of exit code configuration dicts, then use that
+    configuration to decide what's an error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
     upstream process.
     '''
-    log_command(full_command, input_file, output_file)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, output_file, environment)
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
@@ -310,12 +419,14 @@ def execute_command_with_processes(
             command,
             stdin=input_file,
             stdout=None if do_not_capture else (output_file or subprocess.PIPE),
-            stderr=None
-            if do_not_capture
-            else (subprocess.PIPE if output_file else subprocess.STDOUT),
+            stderr=(
+                None if do_not_capture else (subprocess.PIPE if output_file else subprocess.STDOUT)
+            ),
             shell=shell,
             env=environment,
             cwd=working_directory,
+            # Necessary for passing credentials via anonymous pipe.
+            close_fds=False,
         )
     except (subprocess.CalledProcessError, OSError):
         # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
@@ -326,12 +437,14 @@ def execute_command_with_processes(
                 process.kill()
         raise
 
-    captured_outputs = log_outputs(
-        tuple(processes) + (command_process,),
-        (input_file, output_file),
-        output_log_level,
-        borg_local_path=borg_local_path,
-    )
+    with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
+        captured_outputs = log_outputs(
+            tuple(processes) + (command_process,),
+            (input_file, output_file),
+            output_log_level,
+            borg_local_path,
+            borg_exit_codes,
+        )
 
     if output_log_level is None:
         return captured_outputs.get(command_process)

borgmatic/hooks/command.py

@@ -1,8 +1,10 @@
 import logging
 import os
 import re
+import shlex
+import sys
 
-from borgmatic import execute
+import borgmatic.execute
 
 logger = logging.getLogger(__name__)
 
@@ -10,22 +12,38 @@ logger = logging.getLogger(__name__)
 SOFT_FAIL_EXIT_CODE = 75
 
 
-def interpolate_context(config_filename, hook_description, command, context):
+def interpolate_context(hook_description, command, context):
     '''
     Given a config filename, a hook description, a single hook command, and a dict of context
     names/values, interpolate the values by "{name}" into the command and return the result.
     '''
     for name, value in context.items():
-        command = command.replace(f'{{{name}}}', str(value))
+        command = command.replace(f'{{{name}}}', shlex.quote(str(value)))
 
     for unsupported_variable in re.findall(r'{\w+}', command):
         logger.warning(
-            f"{config_filename}: Variable '{unsupported_variable}' is not supported in {hook_description} hook"
+            f"Variable '{unsupported_variable}' is not supported in {hook_description} hook"
         )
 
     return command
 
 
+def make_environment(current_environment, sys_module=sys):
+    '''
+    Given the existing system environment as a map from environment variable name to value, return a
+    copy of it, augmented with any extra environment variables that should be used when running
+    command hooks.
+    '''
+    environment = dict(current_environment)
+
+    # Detect whether we're running within a PyInstaller bundle. If so, set or clear LD_LIBRARY_PATH
+    # based on the value of LD_LIBRARY_PATH_ORIG. This prevents library version information errors.
+    if getattr(sys_module, 'frozen', False) and hasattr(sys_module, '_MEIPASS'):
+        environment['LD_LIBRARY_PATH'] = environment.get('LD_LIBRARY_PATH_ORIG', '')
+
+    return environment
+
+
 def execute_hook(commands, umask, config_filename, description, dry_run, **context):
     '''
     Given a list of hook commands to execute, a umask to execute with (or None), a config filename,
@@ -38,46 +56,45 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
     Raise subprocesses.CalledProcessError if an error occurs in a hook.
     '''
     if not commands:
-        logger.debug(f'{config_filename}: No commands to run for {description} hook')
+        logger.debug(f'No commands to run for {description} hook')
         return
 
     dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
 
     context['configuration_filename'] = config_filename
-    commands = [
-        interpolate_context(config_filename, description, command, context) for command in commands
-    ]
+    commands = [interpolate_context(description, command, context) for command in commands]
 
     if len(commands) == 1:
-        logger.info(f'{config_filename}: Running command for {description} hook{dry_run_label}')
+        logger.info(f'Running command for {description} hook{dry_run_label}')
     else:
         logger.info(
-            f'{config_filename}: Running {len(commands)} commands for {description} hook{dry_run_label}',
+            f'Running {len(commands)} commands for {description} hook{dry_run_label}',
         )
 
     if umask:
         parsed_umask = int(str(umask), 8)
-        logger.debug(f'{config_filename}: Set hook umask to {oct(parsed_umask)}')
+        logger.debug(f'Set hook umask to {oct(parsed_umask)}')
         original_umask = os.umask(parsed_umask)
     else:
         original_umask = None
 
     try:
         for command in commands:
-            if not dry_run:
-                execute.execute_command(
-                    [command],
-                    output_log_level=logging.ERROR
-                    if description == 'on-error'
-                    else logging.WARNING,
-                    shell=True,
-                )
+            if dry_run:
+                continue
+
+            borgmatic.execute.execute_command(
+                [command],
+                output_log_level=(logging.ERROR if description == 'on-error' else logging.WARNING),
+                shell=True,
+                environment=make_environment(os.environ),
+            )
     finally:
         if original_umask:
             os.umask(original_umask)
 
 
-def considered_soft_failure(config_filename, error):
+def considered_soft_failure(error):
     '''
     Given a configuration filename and an exception object, return whether the exception object
     represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so,
@@ -89,7 +106,7 @@ def considered_soft_failure(config_filename, error):
 
     if exit_code == SOFT_FAIL_EXIT_CODE:
         logger.info(
-            f'{config_filename}: Command hook exited with soft failure exit code ({SOFT_FAIL_EXIT_CODE}); skipping remaining actions',
+            f'Command hook exited with soft failure exit code ({SOFT_FAIL_EXIT_CODE}); skipping remaining repository actions',
         )
         return True
 

borgmatic/hooks/credential/container.py

@@ -0,0 +1,43 @@
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+SECRET_NAME_PATTERN = re.compile(r'^\w+$')
+DEFAULT_SECRETS_DIRECTORY = '/run/secrets'
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a secret name to load, read the secret from the corresponding container secrets file
+    and return it.
+
+    Raise ValueError if the credential parameters is not one element, the secret name is invalid, or
+    the secret file cannot be read.
+    '''
+    try:
+        (secret_name,) = credential_parameters
+    except ValueError:
+        name = ' '.join(credential_parameters)
+
+        raise ValueError(f'Cannot load invalid secret name: "{name}"')
+
+    if not SECRET_NAME_PATTERN.match(secret_name):
+        raise ValueError(f'Cannot load invalid secret name: "{secret_name}"')
+
+    try:
+        with open(
+            os.path.join(
+                config.get('working_directory', ''),
+                (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY),
+                secret_name,
+            )
+        ) as secret_file:
+            return secret_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load secret "{secret_name}" from file: {error.filename}')

borgmatic/hooks/credential/file.py

@@ -0,0 +1,30 @@
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a credential path to load, load the credential from file and return it.
+
+    Raise ValueError if the credential parameters is not one element or the secret file cannot be
+    read.
+    '''
+    try:
+        (credential_path,) = credential_parameters
+    except ValueError:
+        name = ' '.join(credential_parameters)
+
+        raise ValueError(f'Cannot load invalid credential: "{name}"')
+
+    try:
+        with open(
+            os.path.join(config.get('working_directory', ''), credential_path)
+        ) as credential_file:
+            return credential_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load credential file: {error.filename}')

borgmatic/hooks/credential/keepassxc.py

@@ -0,0 +1,42 @@
+import logging
+import os
+import shlex
+
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a KeePassXC database path and an attribute name to load, run keepassxc-cli to fetch
+    the corresponidng KeePassXC credential and return it.
+
+    Raise ValueError if keepassxc-cli can't retrieve the credential.
+    '''
+    try:
+        (database_path, attribute_name) = credential_parameters
+    except ValueError:
+        path_and_name = ' '.join(credential_parameters)
+
+        raise ValueError(
+            f'Cannot load credential with invalid KeePassXC database path and attribute name: "{path_and_name}"'
+        )
+
+    if not os.path.exists(database_path):
+        raise ValueError(
+            f'Cannot load credential because KeePassXC database path does not exist: {database_path}'
+        )
+
+    return borgmatic.execute.execute_command_and_capture_output(
+        tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli')))
+        + (
+            'show',
+            '--show-protected',
+            '--attributes',
+            'Password',
+            database_path,
+            attribute_name,
+        )
+    ).rstrip(os.linesep)

borgmatic/hooks/credential/parse.py

@@ -0,0 +1,124 @@
+import functools
+import re
+import shlex
+
+import borgmatic.hooks.dispatch
+
+IS_A_HOOK = False
+
+
+class Hash_adapter:
+    '''
+    A Hash_adapter instance wraps an unhashable object and pretends it's hashable. This is intended
+    for passing to a @functools.cache-decorated function to prevent it from complaining that an
+    argument is unhashable. It should only be used for arguments that you don't want to actually
+    impact the cache hashing, because Hash_adapter doesn't actually hash the object's contents.
+
+    Example usage:
+
+        @functools.cache
+        def func(a, b):
+            print(a, b.actual_value)
+            return a
+
+        func(5, Hash_adapter({1: 2, 3: 4}))  # Calls func(), prints, and returns.
+        func(5, Hash_adapter({1: 2, 3: 4}))  # Hits the cache and just returns the value.
+        func(5, Hash_adapter({5: 6, 7: 8}))  # Also uses cache, since the Hash_adapter is ignored.
+
+    In the above function, the "b" value is one that has been wrapped with Hash_adappter, and
+    therefore "b.actual_value" is necessary to access the original value.
+    '''
+
+    def __init__(self, actual_value):
+        self.actual_value = actual_value
+
+    def __eq__(self, other):
+        return True
+
+    def __hash__(self):
+        return 0
+
+
+UNHASHABLE_TYPES = (dict, list, set)
+
+
+def cache_ignoring_unhashable_arguments(function):
+    '''
+    A function decorator that caches calls to the decorated function but ignores any unhashable
+    arguments when performing cache lookups. This is intended to be a drop-in replacement for
+    functools.cache.
+
+    Example usage:
+
+        @cache_ignoring_unhashable_arguments
+        def func(a, b):
+            print(a, b)
+            return a
+
+        func(5, {1: 2, 3: 4})  # Calls func(), prints, and returns.
+        func(5, {1: 2, 3: 4})  # Hits the cache and just returns the value.
+        func(5, {5: 6, 7: 8})  # Also uses cache, since the unhashable value (the dict) is ignored.
+    '''
+
+    @functools.cache
+    def cached_function(*args, **kwargs):
+        return function(
+            *(arg.actual_value if isinstance(arg, Hash_adapter) else arg for arg in args),
+            **{
+                key: value.actual_value if isinstance(value, Hash_adapter) else value
+                for (key, value) in kwargs.items()
+            },
+        )
+
+    @functools.wraps(function)
+    def wrapper_function(*args, **kwargs):
+        return cached_function(
+            *(Hash_adapter(arg) if isinstance(arg, UNHASHABLE_TYPES) else arg for arg in args),
+            **{
+                key: Hash_adapter(value) if isinstance(value, UNHASHABLE_TYPES) else value
+                for (key, value) in kwargs.items()
+            },
+        )
+
+    wrapper_function.cache_clear = cached_function.cache_clear
+
+    return wrapper_function
+
+
+CREDENTIAL_PATTERN = re.compile(r'\{credential( +(?P<hook_and_parameters>.*))?\}')
+
+
+@cache_ignoring_unhashable_arguments
+def resolve_credential(value, config):
+    '''
+    Given a configuration value containing a string like "{credential hookname credentialname}" and
+    a configuration dict, resolve the credential by calling the relevant hook to get the actual
+    credential value. If the given value does not actually contain a credential tag, then return it
+    unchanged.
+
+    Cache the value (ignoring the config for purposes of caching), so repeated calls to this
+    function don't need to load the credential repeatedly.
+
+    Raise ValueError if the config could not be parsed or the credential could not be loaded.
+    '''
+    if value is None:
+        return value
+
+    matcher = CREDENTIAL_PATTERN.match(value)
+
+    if not matcher:
+        return value
+
+    hook_and_parameters = matcher.group('hook_and_parameters')
+
+    if not hook_and_parameters:
+        raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
+
+    (hook_name, *credential_parameters) = shlex.split(hook_and_parameters)
+
+    if not credential_parameters:
+        raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
+
+    return borgmatic.hooks.dispatch.call_hook(
+        'load_credential', config, hook_name, tuple(credential_parameters)
+    )

borgmatic/hooks/credential/systemd.py

@@ -0,0 +1,43 @@
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$')
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a credential name to load, read the credential from the corresponding systemd
+    credential file and return it.
+
+    Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the
+    credential name is invalid, or the credential file cannot be read.
+    '''
+    try:
+        (credential_name,) = credential_parameters
+    except ValueError:
+        name = ' '.join(credential_parameters)
+
+        raise ValueError(f'Cannot load invalid credential name: "{name}"')
+
+    credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
+
+    if not credentials_directory:
+        raise ValueError(
+            f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
+        )
+
+    if not CREDENTIAL_NAME_PATTERN.match(credential_name):
+        raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
+
+    try:
+        with open(os.path.join(credentials_directory, credential_name)) as credential_file:
+            return credential_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}')

borgmatic/hooks/data_source/bootstrap.py

@@ -0,0 +1,133 @@
+import glob
+import importlib
+import json
+import logging
+import os
+
+import borgmatic.borg.pattern
+import borgmatic.config.paths
+
+logger = logging.getLogger(__name__)
+
+
+def use_streaming(hook_config, config):  # pragma: no cover
+    '''
+    Return whether dump streaming is used for this hook. (Spoiler: It isn't.)
+    '''
+    return False
+
+
+def dump_data_sources(
+    hook_config,
+    config,
+    config_paths,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
+):
+    '''
+    Given a bootstrap configuration dict, a configuration dict, the borgmatic configuration file
+    paths, the borgmatic runtime directory, the configured patterns, and whether this is a dry run,
+    create a borgmatic manifest file to store the paths of the configuration files used to create
+    the archive. But skip this if the bootstrap store_config_files option is False or if this is a
+    dry run.
+
+    Return an empty sequence, since there are no ongoing dump processes from this hook.
+    '''
+    if hook_config and hook_config.get('store_config_files') is False:
+        return []
+
+    borgmatic_manifest_path = os.path.join(
+        borgmatic_runtime_directory, 'bootstrap', 'manifest.json'
+    )
+
+    if dry_run:
+        return []
+
+    os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
+
+    with open(borgmatic_manifest_path, 'w') as manifest_file:
+        json.dump(
+            {
+                'borgmatic_version': importlib.metadata.version('borgmatic'),
+                'config_paths': config_paths,
+            },
+            manifest_file,
+        )
+
+    patterns.extend(
+        borgmatic.borg.pattern.Pattern(
+            config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK
+        )
+        for config_path in config_paths
+    )
+    patterns.append(
+        borgmatic.borg.pattern.Pattern(
+            os.path.join(borgmatic_runtime_directory, 'bootstrap'),
+            source=borgmatic.borg.pattern.Pattern_source.HOOK,
+        )
+    )
+
+    return []
+
+
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+    '''
+    Given a bootstrap configuration dict, a configuration dict, the borgmatic runtime directory, and
+    whether this is a dry run, then remove the manifest file created above. If this is a dry run,
+    then don't actually remove anything.
+    '''
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    manifest_glob = os.path.join(
+        borgmatic.config.paths.replace_temporary_subdirectory_with_glob(
+            os.path.normpath(borgmatic_runtime_directory),
+        ),
+        'bootstrap',
+    )
+    logger.debug(
+        f'Looking for bootstrap manifest files to remove in {manifest_glob}{dry_run_label}'
+    )
+
+    for manifest_directory in glob.glob(manifest_glob):
+        manifest_file_path = os.path.join(manifest_directory, 'manifest.json')
+        logger.debug(f'Removing bootstrap manifest at {manifest_file_path}{dry_run_label}')
+
+        if dry_run:
+            continue
+
+        try:
+            os.remove(manifest_file_path)
+        except FileNotFoundError:
+            pass
+
+        try:
+            os.rmdir(manifest_directory)
+        except FileNotFoundError:
+            pass
+
+
+def make_data_source_dump_patterns(
+    hook_config, config, borgmatic_runtime_directory, name=None
+):  # pragma: no cover
+    '''
+    Restores are implemented via the separate, purpose-specific "bootstrap" action rather than the
+    generic "restore".
+    '''
+    return ()
+
+
+def restore_data_source_dump(
+    hook_config,
+    config,
+    data_source,
+    dry_run,
+    extract_process,
+    connection_params,
+    borgmatic_runtime_directory,
+):  # pragma: no cover
+    '''
+    Restores are implemented via the separate, purpose-specific "bootstrap" action rather than the
+    generic "restore".
+    '''
+    raise NotImplementedError()

borgmatic/hooks/data_source/btrfs.py

@@ -0,0 +1,341 @@
+import collections
+import glob
+import json
+import logging
+import os
+import shutil
+import subprocess
+
+import borgmatic.borg.pattern
+import borgmatic.config.paths
+import borgmatic.execute
+import borgmatic.hooks.data_source.snapshot
+
+logger = logging.getLogger(__name__)
+
+
+def use_streaming(hook_config, config):  # pragma: no cover
+    '''
+    Return whether dump streaming is used for this hook. (Spoiler: It isn't.)
+    '''
+    return False
+
+
+def get_subvolume_mount_points(findmnt_command):
+    '''
+    Given a findmnt command to run, get all sorted Btrfs subvolume mount points.
+    '''
+    findmnt_output = borgmatic.execute.execute_command_and_capture_output(
+        tuple(findmnt_command.split(' '))
+        + (
+            '-t',  # Filesystem type.
+            'btrfs',
+            '--json',
+            '--list',  # Request a flat list instead of a nested subvolume hierarchy.
+        )
+    )
+
+    try:
+        return tuple(
+            sorted(filesystem['target'] for filesystem in json.loads(findmnt_output)['filesystems'])
+        )
+    except json.JSONDecodeError as error:
+        raise ValueError(f'Invalid {findmnt_command} JSON output: {error}')
+    except KeyError as error:
+        raise ValueError(f'Invalid {findmnt_command} output: Missing key "{error}"')
+
+
+Subvolume = collections.namedtuple('Subvolume', ('path', 'contained_patterns'), defaults=((),))
+
+
+def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
+    '''
+    Given a Btrfs command to run and a sequence of configured patterns, find the intersection
+    between the current Btrfs filesystem and subvolume mount points and the paths of any patterns.
+    The idea is that these pattern paths represent the requested subvolumes to snapshot.
+
+    Only include subvolumes that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then
+    return all subvolumes instead, sorted by path.
+
+    Return the result as a sequence of matching subvolume mount points.
+    '''
+    candidate_patterns = set(patterns or ())
+    subvolumes = []
+
+    # For each subvolume mount point, match it against the given patterns to find the subvolumes to
+    # backup. Sort the subvolumes from longest to shortest mount points, so longer mount points get
+    # a whack at the candidate pattern piñata before their parents do. (Patterns are consumed during
+    # this process, so no two subvolumes end up with the same contained patterns.)
+    for mount_point in reversed(get_subvolume_mount_points(findmnt_command)):
+        subvolumes.extend(
+            Subvolume(mount_point, contained_patterns)
+            for contained_patterns in (
+                borgmatic.hooks.data_source.snapshot.get_contained_patterns(
+                    mount_point, candidate_patterns
+                ),
+            )
+            if patterns is None
+            or any(
+                pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                for pattern in contained_patterns
+            )
+        )
+
+    return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path))
+
+
+BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot-'
+
+
+def make_snapshot_path(subvolume_path):
+    '''
+    Given the path to a subvolume, make a corresponding snapshot path for it.
+    '''
+    return os.path.join(
+        subvolume_path,
+        f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
+        # Included so that the snapshot ends up in the Borg archive at the "original" subvolume path.
+    ) + subvolume_path.rstrip(os.path.sep)
+
+
+def make_snapshot_exclude_pattern(subvolume_path):  # pragma: no cover
+    '''
+    Given the path to a subvolume, make a corresponding exclude pattern for its embedded snapshot
+    path. This is to work around a quirk of Btrfs: If you make a snapshot path as a child directory
+    of a subvolume, then the snapshot's own initial directory component shows up as an empty
+    directory within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and
+    make a snapshot of it at:
+
+        /mnt/.borgmatic-snapshot-1234/mnt
+
+    ... then the snapshot itself will have an empty directory at:
+
+        /mnt/.borgmatic-snapshot-1234/mnt/.borgmatic-snapshot-1234
+
+    So to prevent that from ending up in the Borg archive, this function produces an exclude pattern
+    to exclude that path.
+    '''
+    snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
+
+    return borgmatic.borg.pattern.Pattern(
+        os.path.join(
+            subvolume_path,
+            snapshot_directory,
+            subvolume_path.lstrip(os.path.sep),
+            snapshot_directory,
+        ),
+        borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
+        borgmatic.borg.pattern.Pattern_style.FNMATCH,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
+    )
+
+
+def make_borg_snapshot_pattern(subvolume_path, pattern):
+    '''
+    Given the path to a subvolume and a pattern as a borgmatic.borg.pattern.Pattern instance whose
+    path is inside the subvolume, return a new Pattern with its path rewritten to be in a snapshot
+    path intended for giving to Borg.
+
+    Move any initial caret in a regular expression pattern path to the beginning, so as not to break
+    the regular expression.
+    '''
+    initial_caret = (
+        '^'
+        if pattern.style == borgmatic.borg.pattern.Pattern_style.REGULAR_EXPRESSION
+        and pattern.path.startswith('^')
+        else ''
+    )
+
+    rewritten_path = initial_caret + os.path.join(
+        subvolume_path,
+        f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
+        '.',  # Borg 1.4+ "slashdot" hack.
+        # Included so that the source directory ends up in the Borg archive at its "original" path.
+        pattern.path.lstrip('^').lstrip(os.path.sep),
+    )
+
+    return borgmatic.borg.pattern.Pattern(
+        rewritten_path,
+        pattern.type,
+        pattern.style,
+        pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
+    )
+
+
+def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma: no cover
+    '''
+    Given a Btrfs command to run, the path to a subvolume, and the path for a snapshot, create a new
+    Btrfs snapshot of the subvolume.
+    '''
+    os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True)
+
+    borgmatic.execute.execute_command(
+        tuple(btrfs_command.split(' '))
+        + (
+            'subvolume',
+            'snapshot',
+            '-r',  # Read-only.
+            subvolume_path,
+            snapshot_path,
+        ),
+        output_log_level=logging.DEBUG,
+    )
+
+
+def dump_data_sources(
+    hook_config,
+    config,
+    config_paths,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
+):
+    '''
+    Given a Btrfs configuration dict, a configuration dict, the borgmatic configuration file paths,
+    the borgmatic runtime directory, the configured patterns, and whether this is a dry run,
+    auto-detect and snapshot any Btrfs subvolume mount points listed in the given patterns. Also
+    update those patterns, replacing subvolume mount points with corresponding snapshot directories
+    so they get stored in the Borg archive instead.
+
+    Return an empty sequence, since there are no ongoing dump processes from this hook.
+
+    If this is a dry run, then don't actually snapshot anything.
+    '''
+    dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
+    logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}')
+
+    # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those
+    # patterns that came from actual user configuration (as opposed to, say, other hooks).
+    btrfs_command = hook_config.get('btrfs_command', 'btrfs')
+    findmnt_command = hook_config.get('findmnt_command', 'findmnt')
+    subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns)
+
+    if not subvolumes:
+        logger.warning(f'No Btrfs subvolumes found to snapshot{dry_run_label}')
+
+    # Snapshot each subvolume, rewriting patterns to use their snapshot paths.
+    for subvolume in subvolumes:
+        logger.debug(f'Creating Btrfs snapshot for {subvolume.path} subvolume')
+
+        snapshot_path = make_snapshot_path(subvolume.path)
+
+        if dry_run:
+            continue
+
+        snapshot_subvolume(btrfs_command, subvolume.path, snapshot_path)
+
+        for pattern in subvolume.contained_patterns:
+            snapshot_pattern = make_borg_snapshot_pattern(subvolume.path, pattern)
+
+            # Attempt to update the pattern in place, since pattern order matters to Borg.
+            try:
+                patterns[patterns.index(pattern)] = snapshot_pattern
+            except ValueError:
+                patterns.append(snapshot_pattern)
+
+        patterns.append(make_snapshot_exclude_pattern(subvolume.path))
+
+    return []
+
+
+def delete_snapshot(btrfs_command, snapshot_path):  # pragma: no cover
+    '''
+    Given a Btrfs command to run and the name of a snapshot path, delete it.
+    '''
+    borgmatic.execute.execute_command(
+        tuple(btrfs_command.split(' '))
+        + (
+            'subvolume',
+            'delete',
+            snapshot_path,
+        ),
+        output_log_level=logging.DEBUG,
+    )
+
+
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+    '''
+    Given a Btrfs configuration dict, a configuration dict, the borgmatic runtime directory, and
+    whether this is a dry run, delete any Btrfs snapshots created by borgmatic. If this is a dry run
+    or Btrfs isn't configured in borgmatic's configuration, then don't actually remove anything.
+    '''
+    if hook_config is None:
+        return
+
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    btrfs_command = hook_config.get('btrfs_command', 'btrfs')
+    findmnt_command = hook_config.get('findmnt_command', 'findmnt')
+
+    try:
+        all_subvolumes = get_subvolumes(btrfs_command, findmnt_command)
+    except FileNotFoundError as error:
+        logger.debug(f'Could not find "{error.filename}" command')
+        return
+    except subprocess.CalledProcessError as error:
+        logger.debug(error)
+        return
+
+    # Reversing the sorted subvolumes ensures that we remove longer mount point paths of child
+    # subvolumes before the shorter mount point paths of parent subvolumes.
+    for subvolume in reversed(all_subvolumes):
+        subvolume_snapshots_glob = borgmatic.config.paths.replace_temporary_subdirectory_with_glob(
+            os.path.normpath(make_snapshot_path(subvolume.path)),
+            temporary_directory_prefix=BORGMATIC_SNAPSHOT_PREFIX,
+        )
+
+        logger.debug(
+            f'Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}'
+        )
+
+        for snapshot_path in glob.glob(subvolume_snapshots_glob):
+            if not os.path.isdir(snapshot_path):
+                continue
+
+            logger.debug(f'Deleting Btrfs snapshot {snapshot_path}{dry_run_label}')
+
+            if dry_run:
+                continue
+
+            try:
+                delete_snapshot(btrfs_command, snapshot_path)
+            except FileNotFoundError:
+                logger.debug(f'Could not find "{btrfs_command}" command')
+                return
+            except subprocess.CalledProcessError as error:
+                logger.debug(error)
+                return
+
+            # Remove the snapshot parent directory if it still exists. (It might not exist if the
+            # snapshot was for "/".)
+            snapshot_parent_dir = snapshot_path.rsplit(subvolume.path, 1)[0]
+
+            if os.path.isdir(snapshot_parent_dir):
+                shutil.rmtree(snapshot_parent_dir)
+
+
+def make_data_source_dump_patterns(
+    hook_config, config, borgmatic_runtime_directory, name=None
+):  # pragma: no cover
+    '''
+    Restores aren't implemented, because stored files can be extracted directly with "extract".
+    '''
+    return ()
+
+
+def restore_data_source_dump(
+    hook_config,
+    config,
+    data_source,
+    dry_run,
+    extract_process,
+    connection_params,
+    borgmatic_runtime_directory,
+):  # pragma: no cover
+    '''
+    Restores aren't implemented, because stored files can be extracted directly with "extract".
+    '''
+    raise NotImplementedError()

borgmatic/hooks/data_source/dump.py

@@ -0,0 +1,76 @@
+import fnmatch
+import logging
+import os
+import shutil
+
+logger = logging.getLogger(__name__)
+
+IS_A_HOOK = False
+
+
+def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_name):
+    '''
+    Given a borgmatic runtime directory and a data source hook name, construct a data source dump
+    path.
+    '''
+    return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
+
+
+def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
+    '''
+    Based on the given dump directory path, data source name, hostname, and port, return a filename
+    to use for the data source dump. The hostname defaults to localhost.
+
+    Raise ValueError if the data source name is invalid.
+    '''
+    if os.path.sep in name:
+        raise ValueError(f'Invalid data source name {name}')
+
+    return os.path.join(
+        dump_path, (hostname or 'localhost') + ('' if port is None else f':{port}'), name
+    )
+
+
+def create_parent_directory_for_dump(dump_path):
+    '''
+    Create a directory to contain the given dump path.
+    '''
+    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
+
+
+def create_named_pipe_for_dump(dump_path):
+    '''
+    Create a named pipe at the given dump path.
+    '''
+    create_parent_directory_for_dump(dump_path)
+    os.mkfifo(dump_path, mode=0o600)
+
+
+def remove_data_source_dumps(dump_path, data_source_type_name, dry_run):
+    '''
+    Remove all data source dumps in the given dump directory path (including the directory itself).
+    If this is a dry run, then don't actually remove anything.
+    '''
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    logger.debug(f'Removing {data_source_type_name} data source dumps{dry_run_label}')
+
+    if dry_run:
+        return
+
+    if os.path.exists(dump_path):
+        shutil.rmtree(dump_path)
+
+
+def convert_glob_patterns_to_borg_pattern(patterns):
+    '''
+    Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg
+    regular expression archive pattern as a single string like "re:etc/.*|tmp/.*".
+    '''
+    # Remove the "\Z" generated by fnmatch.translate() because we don't want the pattern to match
+    # only at the end of a path, as directory format dumps require extracting files with paths
+    # longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match
+    # paths like "borgmatic/*/foo_databases/test/toc.dat"
+    return 're:' + '|'.join(
+        fnmatch.translate(pattern.lstrip('/')).replace('\\Z', '') for pattern in patterns
+    )

borgmatic/hooks/data_source/lvm.py

@@ -0,0 +1,456 @@
+import collections
+import glob
+import hashlib
+import json
+import logging
+import os
+import shutil
+import subprocess
+
+import borgmatic.borg.pattern
+import borgmatic.config.paths
+import borgmatic.execute
+import borgmatic.hooks.data_source.snapshot
+
+logger = logging.getLogger(__name__)
+
+
+def use_streaming(hook_config, config):  # pragma: no cover
+    '''
+    Return whether dump streaming is used for this hook. (Spoiler: It isn't.)
+    '''
+    return False
+
+
+BORGMATIC_SNAPSHOT_PREFIX = 'borgmatic-'
+Logical_volume = collections.namedtuple(
+    'Logical_volume', ('name', 'device_path', 'mount_point', 'contained_patterns')
+)
+
+
+def get_logical_volumes(lsblk_command, patterns=None):
+    '''
+    Given an lsblk command to run and a sequence of configured patterns, find the intersection
+    between the current LVM logical volume mount points and the paths of any patterns. The idea is
+    that these pattern paths represent the requested logical volumes to snapshot.
+
+    Only include logical volumes that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, include
+    all logical volume mounts points instead, not just those in patterns.
+
+    Return the result as a sequence of Logical_volume instances.
+    '''
+    try:
+        devices_info = json.loads(
+            borgmatic.execute.execute_command_and_capture_output(
+                # Use lsblk instead of lvs here because lvs can't show active mounts.
+                tuple(lsblk_command.split(' '))
+                + (
+                    '--output',
+                    'name,path,mountpoint,type',
+                    '--json',
+                    '--list',
+                )
+            )
+        )
+    except json.JSONDecodeError as error:
+        raise ValueError(f'Invalid {lsblk_command} JSON output: {error}')
+
+    candidate_patterns = set(patterns or ())
+
+    try:
+        # Sort from longest to shortest mount points, so longer mount points get a whack at the
+        # candidate pattern piñata before their parents do. (Patterns are consumed below, so no two
+        # logical volumes end up with the same contained patterns.)
+        return tuple(
+            Logical_volume(device['name'], device['path'], device['mountpoint'], contained_patterns)
+            for device in sorted(
+                devices_info['blockdevices'],
+                key=lambda device: device['mountpoint'] or '',
+                reverse=True,
+            )
+            if device['mountpoint'] and device['type'] == 'lvm'
+            for contained_patterns in (
+                borgmatic.hooks.data_source.snapshot.get_contained_patterns(
+                    device['mountpoint'], candidate_patterns
+                ),
+            )
+            if not patterns
+            or any(
+                pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                for pattern in contained_patterns
+            )
+        )
+    except KeyError as error:
+        raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"')
+
+
+def snapshot_logical_volume(
+    lvcreate_command,
+    snapshot_name,
+    logical_volume_device,
+    snapshot_size,
+):
+    '''
+    Given an lvcreate command to run, a snapshot name, the path to the logical volume device to
+    snapshot, and a snapshot size string, create a new LVM snapshot.
+    '''
+    borgmatic.execute.execute_command(
+        tuple(lvcreate_command.split(' '))
+        + (
+            '--snapshot',
+            ('--extents' if '%' in snapshot_size else '--size'),
+            snapshot_size,
+            '--permission',
+            'r',  # Read-only.
+            '--name',
+            snapshot_name,
+            logical_volume_device,
+        ),
+        output_log_level=logging.DEBUG,
+    )
+
+
+def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # pragma: no cover
+    '''
+    Given a mount command to run, the device path for an existing snapshot, and the path where the
+    snapshot should be mounted, mount the snapshot as read-only (making any necessary directories
+    first).
+    '''
+    os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
+
+    borgmatic.execute.execute_command(
+        tuple(mount_command.split(' '))
+        + (
+            '-o',
+            'ro',
+            snapshot_device,
+            snapshot_mount_path,
+        ),
+        output_log_level=logging.DEBUG,
+    )
+
+
+MOUNT_POINT_HASH_LENGTH = 10
+
+
+def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory):
+    '''
+    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume
+    containing it, return a new Pattern with its path rewritten to be in a snapshot directory based
+    on both the given runtime directory and the given Logical_volume's mount point.
+
+    Move any initial caret in a regular expression pattern path to the beginning, so as not to break
+    the regular expression.
+    '''
+    initial_caret = (
+        '^'
+        if pattern.style == borgmatic.borg.pattern.Pattern_style.REGULAR_EXPRESSION
+        and pattern.path.startswith('^')
+        else ''
+    )
+
+    rewritten_path = initial_caret + os.path.join(
+        normalized_runtime_directory,
+        'lvm_snapshots',
+        # Including this hash prevents conflicts between snapshot patterns for different logical
+        # volumes. For instance, without this, snapshotting a logical volume at /var and another at
+        # /var/spool would result in overlapping snapshot patterns and therefore colliding mount
+        # attempts.
+        hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
+            MOUNT_POINT_HASH_LENGTH
+        ),
+        '.',  # Borg 1.4+ "slashdot" hack.
+        # Included so that the source directory ends up in the Borg archive at its "original" path.
+        pattern.path.lstrip('^').lstrip(os.path.sep),
+    )
+
+    return borgmatic.borg.pattern.Pattern(
+        rewritten_path,
+        pattern.type,
+        pattern.style,
+        pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
+    )
+
+
+DEFAULT_SNAPSHOT_SIZE = '10%ORIGIN'
+
+
+def dump_data_sources(
+    hook_config,
+    config,
+    config_paths,
+    borgmatic_runtime_directory,
+    patterns,
+    dry_run,
+):
+    '''
+    Given an LVM configuration dict, a configuration dict, the borgmatic configuration file paths,
+    the borgmatic runtime directory, the configured patterns, and whether this is a dry run,
+    auto-detect and snapshot any LVM logical volume mount points listed in the given patterns. Also
+    update those patterns, replacing logical volume mount points with corresponding snapshot
+    directories so they get stored in the Borg archive instead.
+
+    Return an empty sequence, since there are no ongoing dump processes from this hook.
+
+    If this is a dry run, then don't actually snapshot anything.
+    '''
+    dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
+    logger.info(f'Snapshotting LVM logical volumes{dry_run_label}')
+
+    # List logical volumes to get their mount points, but only consider those patterns that came
+    # from actual user configuration (as opposed to, say, other hooks).
+    lsblk_command = hook_config.get('lsblk_command', 'lsblk')
+    requested_logical_volumes = get_logical_volumes(lsblk_command, patterns)
+
+    # Snapshot each logical volume, rewriting source directories to use the snapshot paths.
+    snapshot_suffix = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
+    normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory)
+
+    if not requested_logical_volumes:
+        logger.warning(f'No LVM logical volumes found to snapshot{dry_run_label}')
+
+    for logical_volume in requested_logical_volumes:
+        snapshot_name = f'{logical_volume.name}_{snapshot_suffix}'
+        logger.debug(
+            f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}'
+        )
+
+        if not dry_run:
+            snapshot_logical_volume(
+                hook_config.get('lvcreate_command', 'lvcreate'),
+                snapshot_name,
+                logical_volume.device_path,
+                hook_config.get('snapshot_size', DEFAULT_SNAPSHOT_SIZE),
+            )
+
+        # Get the device path for the snapshot we just created.
+        try:
+            snapshot = get_snapshots(
+                hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name
+            )[0]
+        except IndexError:
+            raise ValueError(f'Cannot find LVM snapshot {snapshot_name}')
+
+        # Mount the snapshot into a particular named temporary directory so that the snapshot ends
+        # up in the Borg archive at the "original" logical volume mount point path.
+        snapshot_mount_path = os.path.join(
+            normalized_runtime_directory,
+            'lvm_snapshots',
+            hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
+                MOUNT_POINT_HASH_LENGTH
+            ),
+            logical_volume.mount_point.lstrip(os.path.sep),
+        )
+
+        logger.debug(
+            f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}'
+        )
+
+        if dry_run:
+            continue
+
+        mount_snapshot(
+            hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path
+        )
+
+        for pattern in logical_volume.contained_patterns:
+            snapshot_pattern = make_borg_snapshot_pattern(
+                pattern, logical_volume, normalized_runtime_directory
+            )
+
+            # Attempt to update the pattern in place, since pattern order matters to Borg.
+            try:
+                patterns[patterns.index(pattern)] = snapshot_pattern
+            except ValueError:
+                patterns.append(snapshot_pattern)
+
+    return []
+
+
+def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
+    '''
+    Given a umount command to run and the mount path of a snapshot, unmount it.
+    '''
+    borgmatic.execute.execute_command(
+        tuple(umount_command.split(' ')) + (snapshot_mount_path,),
+        output_log_level=logging.DEBUG,
+    )
+
+
+def remove_snapshot(lvremove_command, snapshot_device_path):  # pragma: no cover
+    '''
+    Given an lvremove command to run and the device path of a snapshot, remove it it.
+    '''
+    borgmatic.execute.execute_command(
+        tuple(lvremove_command.split(' '))
+        + (
+            '--force',  # Suppress an interactive "are you sure?" type prompt.
+            snapshot_device_path,
+        ),
+        output_log_level=logging.DEBUG,
+    )
+
+
+Snapshot = collections.namedtuple(
+    'Snapshot',
+    ('name', 'device_path'),
+)
+
+
+def get_snapshots(lvs_command, snapshot_name=None):
+    '''
+    Given an lvs command to run, return all LVM snapshots as a sequence of Snapshot instances.
+
+    If a snapshot name is given, filter the results to that snapshot.
+    '''
+    try:
+        snapshot_info = json.loads(
+            borgmatic.execute.execute_command_and_capture_output(
+                # Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
+                tuple(lvs_command.split(' '))
+                + (
+                    '--report-format',
+                    'json',
+                    '--options',
+                    'lv_name,lv_path',
+                    '--select',
+                    'lv_attr =~ ^s',  # Filter to just snapshots.
+                )
+            )
+        )
+    except json.JSONDecodeError as error:
+        raise ValueError(f'Invalid {lvs_command} JSON output: {error}')
+
+    try:
+        return tuple(
+            Snapshot(snapshot['lv_name'], snapshot['lv_path'])
+            for snapshot in snapshot_info['report'][0]['lv']
+            if snapshot_name is None or snapshot['lv_name'] == snapshot_name
+        )
+    except IndexError:
+        raise ValueError(f'Invalid {lvs_command} output: Missing report data')
+    except KeyError as error:
+        raise ValueError(f'Invalid {lvs_command} output: Missing key "{error}"')
+
+
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+    '''
+    Given an LVM configuration dict, a configuration dict, the borgmatic runtime directory, and
+    whether this is a dry run, unmount and delete any LVM snapshots created by borgmatic. If this is
+    a dry run or LVM isn't configured in borgmatic's configuration, then don't actually remove
+    anything.
+    '''
+    if hook_config is None:
+        return
+
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    # Unmount snapshots.
+    try:
+        logical_volumes = get_logical_volumes(hook_config.get('lsblk_command', 'lsblk'))
+    except FileNotFoundError as error:
+        logger.debug(f'Could not find "{error.filename}" command')
+        return
+    except subprocess.CalledProcessError as error:
+        logger.debug(error)
+        return
+
+    snapshots_glob = os.path.join(
+        borgmatic.config.paths.replace_temporary_subdirectory_with_glob(
+            os.path.normpath(borgmatic_runtime_directory),
+        ),
+        'lvm_snapshots',
+        '*',
+    )
+    logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
+    umount_command = hook_config.get('umount_command', 'umount')
+
+    for snapshots_directory in glob.glob(snapshots_glob):
+        if not os.path.isdir(snapshots_directory):
+            continue
+
+        for logical_volume in logical_volumes:
+            snapshot_mount_path = os.path.join(
+                snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep)
+            )
+
+            # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
+            # logical volume and therefore there's nothing to unmount.
+            if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
+                continue
+
+            # This might fail if the directory is already mounted, but we swallow errors here since
+            # we'll do another recursive delete below. The point of doing it here is that we don't
+            # want to try to unmount a non-mounted directory (which *will* fail).
+            if not dry_run:
+                shutil.rmtree(snapshot_mount_path, ignore_errors=True)
+
+                # If the delete was successful, that means there's nothing to unmount.
+                if not os.path.isdir(snapshot_mount_path):
+                    continue
+
+            logger.debug(f'Unmounting LVM snapshot at {snapshot_mount_path}{dry_run_label}')
+
+            if dry_run:
+                continue
+
+            try:
+                unmount_snapshot(umount_command, snapshot_mount_path)
+            except FileNotFoundError:
+                logger.debug(f'Could not find "{umount_command}" command')
+                return
+            except subprocess.CalledProcessError as error:
+                logger.debug(error)
+                continue
+
+        if not dry_run:
+            shutil.rmtree(snapshots_directory)
+
+    # Delete snapshots.
+    lvremove_command = hook_config.get('lvremove_command', 'lvremove')
+
+    try:
+        snapshots = get_snapshots(hook_config.get('lvs_command', 'lvs'))
+    except FileNotFoundError as error:
+        logger.debug(f'Could not find "{error.filename}" command')
+        return
+    except subprocess.CalledProcessError as error:
+        logger.debug(error)
+        return
+
+    for snapshot in snapshots:
+        # Only delete snapshots that borgmatic actually created!
+        if not snapshot.name.split('_')[-1].startswith(BORGMATIC_SNAPSHOT_PREFIX):
+            continue
+
+        logger.debug(f'Deleting LVM snapshot {snapshot.name}{dry_run_label}')
+
+        if not dry_run:
+            remove_snapshot(lvremove_command, snapshot.device_path)
+
+
+def make_data_source_dump_patterns(
+    hook_config, config, borgmatic_runtime_directory, name=None
+):  # pragma: no cover
+    '''
+    Restores aren't implemented, because stored files can be extracted directly with "extract".
+    '''
+    return ()
+
+
+def restore_data_source_dump(
+    hook_config,
+    config,
+    data_source,
+    dry_run,
+    extract_process,
+    connection_params,
+    borgmatic_runtime_directory,
+):  # pragma: no cover
+    '''
+    Restores aren't implemented, because stored files can be extracted directly with "extract".
+    '''
+    raise NotImplementedError()