Merge branch 'main' into command-hook-steps

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1121
Reviewed-by: Dan Helfman <witten@torsion.org>

.bandit

@@ -0,0 +1,3 @@
+[bandit]
+exclude=tests
+skips=S105,S404

.gitea/pull_request_template.md

@@ -0,0 +1,4 @@
+---
+name: "Pull Request"
+about: "Pull Request"
+---

.gitea/workflows/build.yaml

@@ -17,6 +17,7 @@ jobs:
   docs:
     needs: [test]
     runs-on: host
+    if: gitea.event_name == 'push'
     env:
       IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
 

.github/pull_request_template.md

@@ -0,0 +1,10 @@
+## Hold up, GitHub users
+
+Thanks for your contribution! 
+
+Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see:
+
+https://torsion.org/borgmatic/#contributing
+
+... which provides full instructions on how to submit pull requests. You can even use your GitHub account to login.
+

MANIFEST.in

@@ -1,2 +1,7 @@
+# This file only applies to the source dist tarball, not the built wheel.
 include borgmatic/config/schema.yaml
-graft sample/systemd
+graft docs
+graft sample
+graft scripts
+graft tests
+global-exclude *.py[co]

NEWS

@@ -1,3 +1,107 @@
+2.0.9.dev0
+ * #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd
+   service.
+ * #1134: Add a "steps" option to run command hooks around particular sub-action steps like
+   individual checks.
+ * #1149: Add support for Python 3.14.
+ * #1149: Include automated tests in the source dist tarball uploaded to PyPI.
+ * #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+
+   "slashdot" hack within source directory paths.
+
+2.0.8
+ * #1114: Document systemd configuration changes for the ZFS filesystem hook.
+ * #1116: Add dumping of database containers via their container names, handy for backing up
+   database containers from the host. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-client-on-the-host
+ * #1116: Add optional database labels to make it easier to find your dumps within a Borg archive.
+ * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in
+   use.
+ * #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always
+   error and exit when the borgmatic runtime directory overlaps with the configured excludes.
+ * #1125: Clarify documentation about ZFS, Btrfs, and LVM snapshotting when a separate
+   filesystem is mounted in the source directory. (Spoiler: The separate filesystem doesn't get
+   included in the snapshot.)
+ * #1126: Create LVM snapshots as read-write to avoid an error when snapshotting ext4 filesystems
+   with orphaned files that need recovery.
+ * #1133: Fix the "spot" check to include borgmatic configuration files that were backed up to
+   support the "bootstrap" action.
+ * #1136: For all database hooks, record metadata about the dumps contained within an archive.
+ * #1139: Set "borgmatic" as the user agent when connecting to monitoring services.
+ * #1146: Fix a broken "create" action and "--archive latest" flag when multiple archives share the
+   same name with Borg 2.
+ * Treat configuration file permissions issues as errors instead of warnings.
+ * When running tests, use Ruff for faster and more comprehensive code linting and formatting,
+   replacing Flake8, Black, isort, etc.
+ * Switch from pipx to uv for installing development tools, and added tox-uv for speeding up test
+   environment creation. See the developer documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
+
+2.0.7
+ * #1032: Fix a bug in which a Borg archive gets created even when a database hook fails.
+ * #1043: Support Btrfs subvolume paths in "source_directories" even when the subvolume is mounted
+   elsewhere.
+ * #1048: Ignore Btrfs subvolumes whose read-only status can't be determined.
+ * #1083: Add "debug_passphrase"/"display_passphrase" options and a "{unixtime}" placeholder in
+   support of Borg 2 features.
+ * #1099: Clarify documentation on command hooks order of execution.
+ * #1100: Fix a bug in which "borg --version" failing isn't considered a "fail" state in a command
+   hook.
+ * #1108: Fix a bug in which quoted "extra_borg_options" values containing spaces are passed to Borg
+   incorrectly.
+ * #1108: Add a "--comment" flag to the "create" action for creating an archive with a comment.
+ * Use the Bandit security analysis tool when running tests.
+ * SECURITY: Add timeouts to all monitoring hooks to prevent hangs on network requests, e.g. due to
+   a compromised monitoring server holding requests open.
+ * SECURITY: For the "spot" check, use a more secure source of randomness when selecting paths to
+   check.
+
+2.0.6
+ * #1068: Fix a warning from LVM about leaked file descriptors.
+ * #1086: Fix for the "spot" check breaking when the "--progress" flag is used.
+ * #1089: Fix for the "spot" check erroring when a checksum command errors.
+ * #1091: Fix for the "config generate" action generating invalid configuration when upgrading
+   deprecated command hooks.
+ * #1093: Fix for the LVM hook erroring when the "--dry-run" flag is used.
+ * #1094: Fix incorrect documentation about customizing Borg exit codes:
+   https://torsion.org/borgmatic/docs/how-to/customize-warnings-and-errors/
+ * #1095: Fix for the "spot" check's "xxh64sum_command" option erroring on commands containing
+   spaces.
+ * Add support for Borg 2's "s3:" and "b2:" repository URLs, so you can backup to S3 or B2 cloud
+   storage services even without using Rclone.
+ * During the "spot" check, truncate log messages containing many file paths.
+
+2.0.5
+ * #1033: Add a "password_transport" option to the MariaDB and MySQL database hooks for customizing
+   how borgmatic transmits passwords to the database client.
+ * #1078: Add "keep_3monthly" and "keep_13weekly" options for customizing "prune" action archive
+   retention.
+ * #1078: Add a "use_chunks_archive" option for controlling whether Borg uses its chunks cache
+   directory.
+ * #1078: For the "compact" action, pass "--dry-run" through to Borg.
+ * #1085: Fix a regression in which the default monitoring verbosity is 0 (warnings only) instead of
+   1 (info about steps borgmatic is taking). This prevented logs from showing up in monitoring
+   services like Healthchecks unless you had an explicit monitoring verbosity set.
+ * Move Mastodon social hosting from Fosstodon to FLOSS.social: https://floss.social/@borgmatic
+ * The borgmatic project no longer accepts pull requests on GitHub. But see
+   https://torsion.org/borgmatic/#contributing for how you can still submit pull requests. You can
+   even use your GitHub account to login.
+
+2.0.4
+ * #1072: Fix path rewriting for non-root patterns in the ZFS, Btrfs, and LVM hooks.
+ * #1073: Clarify the documentation about when an "after: error" command hook runs and how it
+   differs from other hooks:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #1075: Fix an incorrect warning about Borg placeholders being unsupported in a command hook.
+ * #1080: If the exact same "everything" command hook is present in multiple configuration files,
+   only run it once.
+
+2.0.3
+ * #1065: Fix a regression in monitoring hooks in which an error pinged the finish state instead of
+   the fail state.
+ * #1066: Add a "states" option to command hooks, so you can optionally skip an "after" hook if
+   borgmatic encounters an error.
+ * #1071: Fix an error in the LVM hook when removing a snapshot directory.
+
 2.0.2
  * #1035: Document potential performance issues and workarounds with the ZFS, Btrfs, and LVM hooks:
    https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
@@ -46,7 +150,7 @@
  * #837: Add custom command options for the MongoDB hook.
  * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune".
  * #1020: Document a database use case involving a temporary database client container:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
  * #1037: Fix an error with the "extract" action when both a remote repository and a
    "working_directory" are used.
  * #1044: Fix an error in the systemd credential hook when the credential name contains a "."
@@ -296,7 +400,7 @@
    paths when a "working_directory" is set.
  * #906: Add documentation details for how to run custom database dump commands using binaries from
    running containers:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
  * Fix a regression in which the "color" option had no effect.
  * Add a recent contributors section to the documentation, because credit where credit's due! See:
    https://torsion.org/borgmatic/#recent-contributors
@@ -352,7 +456,7 @@
  * Fix handling of the NO_COLOR environment variable to ignore an empty value.
  * Add documentation about backing up containerized databases by configuring borgmatic to exec into
    a container to run a dump command:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
 
 1.8.9
  * #311: Add custom dump/restore command options for MySQL and MariaDB.
@@ -622,7 +726,7 @@
    at the command-line. See the configuration reference for more information:
    https://torsion.org/borgmatic/docs/reference/configuration/
  * #649: Add documentation on backing up a database running in a container:
-   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
  * #655: Fix error when databases are configured and a source directory doesn't exist.
  * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables.
    To join in the pedantry, refresh your test environment with "tox --recreate".

README.md

@@ -138,7 +138,8 @@ issues.
 
 ### Social
 
-Follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
+Follow <a rel="me" href="https://floss.social/@borgmatic">borgmatic on
+Mastodon</a>.
 
 
 ### Chat
@@ -159,21 +160,21 @@ Other questions or comments? Contact
 ### Contributing
 
 borgmatic [source code is
-available](https://projects.torsion.org/borgmatic-collective/borgmatic) and is also mirrored
+available](https://projects.torsion.org/borgmatic-collective/borgmatic) and also
-on [GitHub](https://github.com/borgmatic-collective/borgmatic) for convenience.
+has a read-only mirror on
+[GitHub](https://github.com/borgmatic-collective/borgmatic) for convenience.
 
 borgmatic is licensed under the GNU General Public License version 3 or any
 later version.
 
-If you'd like to contribute to borgmatic development, please feel free to
+If you'd like to contribute to borgmatic development, please feel free to submit
-submit a [Pull
+a [pull
-Request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls) or
+request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls) or
 open an
 [issue](https://projects.torsion.org/borgmatic-collective/borgmatic/issues) to
 discuss your idea. Note that you'll need to
 [register](https://projects.torsion.org/user/sign_up?invite_code=borgmatic)
-first. We also accept Pull Requests on GitHub, if that's more your thing. In
+first. In general, contributions are very welcome. We don't bite!
-general, contributions are very welcome. We don't bite!
 
 Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for

borgmatic/actions/borg.py

@@ -20,7 +20,8 @@ def run_borg(
     Run the "borg" action for the given repository.
     '''
     if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, borg_arguments.repository
+        repository,
+        borg_arguments.repository,
     ):
         logger.info('Running arbitrary Borg command')
         archive_name = borgmatic.borg.repo_list.resolve_archive_name(

borgmatic/actions/break_lock.py

@@ -19,7 +19,8 @@ def run_break_lock(
     Run the "break-lock" action for the given repository.
     '''
     if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, break_lock_arguments.repository
+        repository,
+        break_lock_arguments.repository,
     ):
         logger.info('Breaking repository and cache locks')
         borgmatic.borg.break_lock.break_lock(

borgmatic/actions/change_passphrase.py

@@ -21,7 +21,8 @@ def run_change_passphrase(
     if (
         change_passphrase_arguments.repository is None
         or borgmatic.config.validate.repositories_match(
-            repository, change_passphrase_arguments.repository
+            repository,
+            change_passphrase_arguments.repository,
         )
     ):
         logger.info('Changing repository passphrase')

borgmatic/actions/check.py

@@ -1,4 +1,5 @@
 import calendar
+import contextlib
 import datetime
 import hashlib
 import itertools
@@ -6,14 +7,18 @@ import logging
 import os
 import pathlib
 import random
+import shlex
 import shutil
+import textwrap
 
+import borgmatic.actions.config.bootstrap
 import borgmatic.actions.pattern
 import borgmatic.borg.check
 import borgmatic.borg.create
 import borgmatic.borg.environment
 import borgmatic.borg.extract
 import borgmatic.borg.list
+import borgmatic.borg.pattern
 import borgmatic.borg.repo_list
 import borgmatic.borg.state
 import borgmatic.config.paths
@@ -53,12 +58,14 @@ def parse_checks(config, only_checks=None):
 
     if 'disabled' in checks:
         logger.warning(
-            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
+            'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead',
         )
+
         if len(checks) > 1:
             logger.warning(
-                'Multiple checks are configured, but one of them is "disabled"; not running any checks'
+                'Multiple checks are configured, but one of them is "disabled"; not running any checks',
             )
+
         return ()
 
     return checks
@@ -162,6 +169,7 @@ def filter_checks_on_frequency(
                     **dict.fromkeys(day for day in days if day != 'weekday'),
                     **dict.fromkeys(WEEKDAY_DAYS),
                 }
+
             if 'weekend' in days:
                 days = {
                     **dict.fromkeys(day for day in days if day != 'weekend'),
@@ -170,7 +178,7 @@ def filter_checks_on_frequency(
 
             if calendar.day_name[datetime_now().weekday()] not in days:
                 logger.info(
-                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)"
+                    f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)",
                 )
                 filtered_checks.remove(check)
                 continue
@@ -188,7 +196,7 @@ def filter_checks_on_frequency(
         if datetime_now() < check_time + frequency_delta:
             remaining = check_time + frequency_delta - datetime_now()
             logger.info(
-                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
+                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)',
             )
             filtered_checks.remove(check)
 
@@ -214,7 +222,7 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
     '''
     borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
 
-    if check_type in ('archives', 'data'):
+    if check_type in {'archives', 'data'}:
         return os.path.join(
             borgmatic_state_directory,
             'checks',
@@ -249,7 +257,7 @@ def read_check_time(path):
     logger.debug(f'Reading check time from {path}')
 
     try:
-        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
+        return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)  # noqa: DTZ006
     except FileNotFoundError:
         return None
 
@@ -280,7 +288,7 @@ def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
             (
                 make_check_time_path(config, borg_repository_id, check, archives_check_id),
                 make_check_time_path(config, borg_repository_id, check),
-            )
+            ),
         )
     )
 
@@ -312,16 +320,17 @@ def upgrade_check_times(config, borg_repository_id):
       {borgmatic_state_directory}/checks/1234567890/archives/all
     '''
     borgmatic_source_checks_path = os.path.join(
-        borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
+        borgmatic.config.paths.get_borgmatic_source_directory(config),
+        'checks',
     )
     borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
     borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
 
     if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
-        borgmatic_state_checks_path
+        borgmatic_state_checks_path,
     ):
         logger.debug(
-            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
+            f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}',
         )
         os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
         shutil.move(borgmatic_source_checks_path, borgmatic_state_checks_path)
@@ -336,10 +345,8 @@ def upgrade_check_times(config, borg_repository_id):
 
         logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
 
-        try:
+        with contextlib.suppress(FileNotFoundError):
             shutil.move(old_path, temporary_path)
-        except FileNotFoundError:
-            pass
 
         os.mkdir(old_path)
         shutil.move(temporary_path, new_path)
@@ -353,37 +360,51 @@ def collect_spot_check_source_paths(
     local_path,
     remote_path,
     borgmatic_runtime_directory,
+    bootstrap_config_paths,
 ):
     '''
     Given a repository configuration dict, a configuration dict, the local Borg version, global
-    arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
+    arguments as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the
-    collect the source paths that Borg would use in an actual create (but only include files).
+    bootstrap configuration paths as read from an archive's manifest, collect the source paths that
+    Borg would use in an actual create (but only include files). As part of this, include the
+    bootstrap configuration paths, so that any configuration files included in the archive to
+    support bootstrapping are also spot checked.
     '''
     stream_processes = any(
         borgmatic.hooks.dispatch.call_hooks(
             'use_streaming',
             config,
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
-        ).values()
+        ).values(),
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, pattern_file) = (
+    (create_flags, create_positional_arguments, _) = borgmatic.borg.create.make_base_create_command(
-        borgmatic.borg.create.make_base_create_command(
+        dry_run=True,
-            dry_run=True,
+        repository_path=repository['path'],
-            repository_path=repository['path'],
+        # Omit "progress" because it interferes with "list_details".
-            config=dict(config, list_details=True),
+        config=dict(
-            patterns=borgmatic.actions.pattern.process_patterns(
+            {option: value for option, value in config.items() if option != 'progress'},
-                borgmatic.actions.pattern.collect_patterns(config),
+            list_details=True,
-                working_directory,
+        ),
+        patterns=borgmatic.actions.pattern.process_patterns(
+            borgmatic.actions.pattern.collect_patterns(config)
+            + tuple(
+                borgmatic.borg.pattern.Pattern(
+                    config_path,
+                    source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
+                )
+                for config_path in bootstrap_config_paths
             ),
-            local_borg_version=local_borg_version,
+            config,
-            global_arguments=global_arguments,
+            working_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
+        ),
-            local_path=local_path,
+        local_borg_version=local_borg_version,
-            remote_path=remote_path,
+        global_arguments=global_arguments,
-            stream_processes=stream_processes,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
-        )
+        local_path=local_path,
+        remote_path=remote_path,
+        stream_processes=stream_processes,
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
@@ -399,7 +420,7 @@ def collect_spot_check_source_paths(
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.splitlines()
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
+        if path_line and path_line.startswith(('- ', '+ '))
     )
 
     return tuple(
@@ -440,12 +461,12 @@ def collect_spot_check_archive_paths(
             config,
             local_borg_version,
             global_arguments,
-            path_format='{type} {path}{NUL}',  # noqa: FS003
+            path_format='{type} {path}{NUL}',
             local_path=local_path,
             remote_path=remote_path,
         )
         for (file_type, path) in (line.split(' ', 1),)
-        if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
+        if file_type not in {BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE}
         if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
         if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
         not in pathlib.Path(path).parents
@@ -454,7 +475,7 @@ def collect_spot_check_archive_paths(
     )
 
 
-SAMPLE_PATHS_SUBSET_COUNT = 10000
+SAMPLE_PATHS_SUBSET_COUNT = 5000
 
 
 def compare_spot_check_hashes(
@@ -478,9 +499,10 @@ def compare_spot_check_hashes(
     # source directories.
     spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
     sample_count = max(
-        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
+        int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)),
+        1,
     )
-    source_sample_paths = tuple(random.sample(source_paths, sample_count))
+    source_sample_paths = tuple(random.SystemRandom().sample(source_paths, sample_count))
     working_directory = borgmatic.config.paths.get_working_directory(config)
     hashable_source_sample_path = {
         source_path
@@ -490,7 +512,7 @@ def compare_spot_check_hashes(
         if not os.path.islink(full_source_path)
     }
     logger.debug(
-        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
+        f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check',
     )
 
     source_sample_paths_iterator = iter(source_sample_paths)
@@ -502,13 +524,16 @@ def compare_spot_check_hashes(
     while True:
         # Hash each file in the sample paths (if it exists).
         source_sample_paths_subset = tuple(
-            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT)
+            itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT),
         )
         if not source_sample_paths_subset:
             break
 
         hash_output = borgmatic.execute.execute_command_and_capture_output(
-            (spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
+            tuple(
+                shlex.quote(part)
+                for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
+            )
             + tuple(
                 path for path in source_sample_paths_subset if path in hashable_source_sample_path
             ),
@@ -526,7 +551,7 @@ def compare_spot_check_hashes(
                     for path in source_sample_paths_subset
                     if path not in hashable_source_sample_path
                 },
-            )
+            ),
         )
 
         # Get the hash for each file in the archive.
@@ -540,12 +565,12 @@ def compare_spot_check_hashes(
                     local_borg_version,
                     global_arguments,
                     list_paths=source_sample_paths_subset,
-                    path_format='{xxh64} {path}{NUL}',  # noqa: FS003
+                    path_format='{xxh64} {path}{NUL}',
                     local_path=local_path,
                     remote_path=remote_path,
                 )
                 if line
-            )
+            ),
         )
 
     # Compare the source hashes with the archive hashes to see how many match.
@@ -562,6 +587,9 @@ def compare_spot_check_hashes(
     return tuple(failing_paths)
 
 
+MAX_SPOT_CHECK_PATHS_LENGTH = 1000
+
+
 def spot_check(
     repository,
     config,
@@ -591,20 +619,9 @@ def spot_check(
 
     if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
         raise ValueError(
-            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage'
+            'The data_tolerance_percentage must be less than or equal to the data_sample_percentage',
         )
 
-    source_paths = collect_spot_check_source_paths(
-        repository,
-        config,
-        local_borg_version,
-        global_arguments,
-        local_path,
-        remote_path,
-        borgmatic_runtime_directory,
-    )
-    logger.debug(f'{len(source_paths)} total source paths for spot check')
-
     archive = borgmatic.borg.repo_list.resolve_archive_name(
         repository['path'],
         'latest',
@@ -616,6 +633,25 @@ def spot_check(
     )
     logger.debug(f'Using archive {archive} for spot check')
 
+    source_paths = collect_spot_check_source_paths(
+        repository,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        borgmatic_runtime_directory,
+        bootstrap_config_paths=borgmatic.actions.config.bootstrap.load_config_paths_from_archive(
+            repository['path'],
+            archive,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        ),
+    )
+    logger.debug(f'{len(source_paths)} total source paths for spot check')
+
     archive_paths = collect_spot_check_archive_paths(
         repository,
         archive,
@@ -629,11 +665,14 @@ def spot_check(
     logger.debug(f'{len(archive_paths)} total archive paths for spot check')
 
     if len(source_paths) == 0:
-        logger.debug(
+        truncated_archive_paths = textwrap.shorten(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}'
+            ', '.join(set(archive_paths)) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
+        logger.debug(f'Paths in latest archive but not source paths: {truncated_archive_paths}')
         raise ValueError(
-            'Spot check failed: There are no source paths to compare against the archive'
+            'Spot check failed: There are no source paths to compare against the archive',
         )
 
     # Calculate the percentage delta between the source paths count and the archive paths count, and
@@ -641,15 +680,25 @@ def spot_check(
     count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
 
     if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
-        rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
+        rootless_source_paths = {path.lstrip(os.path.sep) for path in source_paths}
-        logger.debug(
+        truncated_exclusive_source_paths = textwrap.shorten(
-            f'Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}'
+            ', '.join(rootless_source_paths - set(archive_paths)) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
         logger.debug(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}'
+            f'Paths in source paths but not latest archive: {truncated_exclusive_source_paths}',
+        )
+        truncated_exclusive_archive_paths = textwrap.shorten(
+            ', '.join(set(archive_paths) - rootless_source_paths) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
+        )
+        logger.debug(
+            f'Paths in latest archive but not source paths: {truncated_exclusive_archive_paths}',
         )
         raise ValueError(
-            f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
+            f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)',
         )
 
     failing_paths = compare_spot_check_hashes(
@@ -669,15 +718,20 @@ def spot_check(
     failing_percentage = (len(failing_paths) / len(source_paths)) * 100
 
     if failing_percentage > data_tolerance_percentage:
+        truncated_failing_paths = textwrap.shorten(
+            ', '.join(failing_paths),
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
+        )
         logger.debug(
-            f'Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
+            f'Source paths with data not matching the latest archive: {truncated_failing_paths}',
         )
         raise ValueError(
-            f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
+            f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)',
         )
 
     logger.info(
-        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
+        f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta',
     )
 
 
@@ -690,6 +744,7 @@ def run_check(
     global_arguments,
     local_path,
     remote_path,
+    hook_context,
 ):
     '''
     Run the "check" action for the given repository.
@@ -697,7 +752,8 @@ def run_check(
     Raise ValueError if the Borg repository ID cannot be determined.
     '''
     if check_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, check_arguments.repository
+        repository,
+        check_arguments.repository,
     ):
         return
 
@@ -714,7 +770,10 @@ def run_check(
     upgrade_check_times(config, repository_id)
     configured_checks = parse_checks(config, check_arguments.only_checks)
     archive_filter_flags = borgmatic.borg.check.make_archive_filter_flags(
-        local_borg_version, config, configured_checks, check_arguments
+        local_borg_version,
+        config,
+        configured_checks,
+        check_arguments,
     )
     archives_check_id = make_archives_check_id(archive_filter_flags)
     checks = filter_checks_on_frequency(
@@ -725,43 +784,75 @@ def run_check(
         archives_check_id,
     )
     borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
+    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     if borg_specific_checks:
-        borgmatic.borg.check.check_archives(
+        with borgmatic.hooks.command.Before_after_hooks(
-            repository['path'],
+            command_hooks=config.get('commands'),
-            config,
+            before_after='step',
-            local_borg_version,
+            umask=config.get('umask'),
-            check_arguments,
+            working_directory=working_directory,
-            global_arguments,
+            dry_run=global_arguments.dry_run,
-            borg_specific_checks,
+            action_names=('check',),
-            archive_filter_flags,
+            step_names=('archives_repository_data',),
-            local_path=local_path,
+            **hook_context,
-            remote_path=remote_path,
+        ):
-        )
+            borgmatic.borg.check.check_archives(
-        for check in borg_specific_checks:
+                repository['path'],
-            write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
+                config,
+                local_borg_version,
+                check_arguments,
+                global_arguments,
+                borg_specific_checks,
+                archive_filter_flags,
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            for check in borg_specific_checks:
+                write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
 
     if 'extract' in checks:
-        borgmatic.borg.extract.extract_last_archive_dry_run(
+        with borgmatic.hooks.command.Before_after_hooks(
-            config,
+            command_hooks=config.get('commands'),
-            local_borg_version,
+            before_after='step',
-            global_arguments,
+            umask=config.get('umask'),
-            repository['path'],
+            working_directory=working_directory,
-            config.get('lock_wait'),
+            dry_run=global_arguments.dry_run,
-            local_path,
+            action_names=('check',),
-            remote_path,
+            step_names=('extract',),
-        )
+            **hook_context,
-        write_check_time(make_check_time_path(config, repository_id, 'extract'))
+        ):
-
+            borgmatic.borg.extract.extract_last_archive_dry_run(
-    if 'spot' in checks:
-        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-            spot_check(
-                repository,
                 config,
                 local_borg_version,
                 global_arguments,
+                repository['path'],
+                config.get('lock_wait'),
                 local_path,
                 remote_path,
-                borgmatic_runtime_directory,
             )
-        write_check_time(make_check_time_path(config, repository_id, 'spot'))
+            write_check_time(make_check_time_path(config, repository_id, 'extract'))
+
+    if 'spot' in checks:
+        with borgmatic.hooks.command.Before_after_hooks(
+            command_hooks=config.get('commands'),
+            before_after='step',
+            umask=config.get('umask'),
+            working_directory=working_directory,
+            dry_run=global_arguments.dry_run,
+            action_names=('check',),
+            step_names=('spot',),
+            **hook_context,
+        ):
+            with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+                spot_check(
+                    repository,
+                    config,
+                    local_borg_version,
+                    global_arguments,
+                    local_path,
+                    remote_path,
+                    borgmatic_runtime_directory,
+                )
+
+            write_check_time(make_check_time_path(config, repository_id, 'spot'))

borgmatic/actions/compact.py

@@ -23,7 +23,8 @@ def run_compact(
     Run the "compact" action for the given repository.
     '''
     if compact_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, compact_arguments.repository
+        repository,
+        compact_arguments.repository,
     ):
         return
 

borgmatic/actions/config/bootstrap.py

@@ -16,78 +16,81 @@ def make_bootstrap_config(bootstrap_arguments):
     Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
     '''
     return {
-        'ssh_command': bootstrap_arguments.ssh_command,
+        'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory,
+        'local_path': bootstrap_arguments.local_path,
+        'remote_path': bootstrap_arguments.remote_path,
         # In case the repo has been moved or is accessed from a different path at the point of
         # bootstrapping.
         'relocated_repo_access_is_ok': True,
+        'ssh_command': bootstrap_arguments.ssh_command,
+        'user_runtime_directory': bootstrap_arguments.user_runtime_directory,
     }
 
 
-def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
+def load_config_paths_from_archive(
+    repository_path,
+    archive_name,
+    config,
+    local_borg_version,
+    global_arguments,
+    borgmatic_runtime_directory,
+):
     '''
-    Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
+    Given a repository path, an archive name, a configuration dict, the local Borg version, the
-    repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
+    global arguments as an argparse.Namespace, and the borgmatic runtime directory, return the
-    borgmatic source directory, destination directory, and whether to strip components), the global
+    config paths from the manifest.json file in the borgmatic source directory or runtime directory
-    arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
+    within the repository archive.
-    return the config paths from the manifest.json file in the borgmatic source directory or runtime
-    directory after extracting it from the repository archive.
 
     Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
     expected configuration path data.
     '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
-        {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
-    )
-    config = make_bootstrap_config(bootstrap_arguments)
-
     # Probe for the manifest file in multiple locations, as the default location has moved to the
     # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
     # still want to support reading the manifest from previously created archives as well.
-    with borgmatic.config.paths.Runtime_directory(
+    for base_directory in (
-        {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
+        'borgmatic',
-    ) as borgmatic_runtime_directory:
+        borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-        for base_directory in (
+        borgmatic.config.paths.get_borgmatic_source_directory(config),
-            'borgmatic',
+    ):
-            borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+        borgmatic_manifest_path = 'sh:' + os.path.join(
-            borgmatic_source_directory,
+            base_directory,
-        ):
+            'bootstrap',
-            borgmatic_manifest_path = 'sh:' + os.path.join(
+            'manifest.json',
-                base_directory, 'bootstrap', 'manifest.json'
+        )
-            )
 
-            extract_process = borgmatic.borg.extract.extract_archive(
+        extract_process = borgmatic.borg.extract.extract_archive(
-                global_arguments.dry_run,
+            global_arguments.dry_run,
-                bootstrap_arguments.repository,
+            repository_path,
-                archive_name,
+            archive_name,
-                [borgmatic_manifest_path],
+            [borgmatic_manifest_path],
-                config,
+            config,
-                local_borg_version,
+            local_borg_version,
-                global_arguments,
+            global_arguments,
-                local_path=bootstrap_arguments.local_path,
+            local_path=config.get('local_path'),
-                remote_path=bootstrap_arguments.remote_path,
+            remote_path=config.get('remote_path'),
-                extract_to_stdout=True,
+            extract_to_stdout=True,
-            )
+        )
-            manifest_json = extract_process.stdout.read()
+        manifest_json = extract_process.stdout.read()
 
-            if manifest_json:
+        if manifest_json:
-                break
+            break
-        else:
+    else:
-            raise ValueError(
+        raise ValueError(
-                'Cannot read configuration paths from archive due to missing bootstrap manifest'
+            'Cannot read configuration paths from archive due to missing archive or bootstrap manifest',
-            )
+        )
 
     try:
         manifest_data = json.loads(manifest_json)
     except json.JSONDecodeError as error:
         raise ValueError(
-            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
+            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}',
         )
 
     try:
         return manifest_data['config_paths']
     except KeyError:
         raise ValueError(
-            'Cannot read configuration paths from archive due to invalid bootstrap manifest'
+            'Cannot read configuration paths from archive due to invalid bootstrap manifest',
         )
 
 
@@ -108,9 +111,16 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
         local_path=bootstrap_arguments.local_path,
         remote_path=bootstrap_arguments.remote_path,
     )
-    manifest_config_paths = get_config_paths(
+
-        archive_name, bootstrap_arguments, global_arguments, local_borg_version
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-    )
+        manifest_config_paths = load_config_paths_from_archive(
+            bootstrap_arguments.repository,
+            archive_name,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        )
 
     logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
 

borgmatic/actions/config/generate.py

@@ -19,7 +19,7 @@ def run_generate(generate_arguments, global_arguments):
     dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
 
     logger.answer(
-        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
+        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}',
     )
 
     borgmatic.config.generate.generate_sample_configuration(
@@ -36,7 +36,7 @@ def run_generate(generate_arguments, global_arguments):
 Merged in the contents of configuration file at: {generate_arguments.source_filename}
 To review the changes made, run:
 
-    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
+    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}''',
         )
 
     logger.answer(
@@ -44,5 +44,5 @@ To review the changes made, run:
 This includes all available configuration options with example values, the few
 required options as indicated. Please edit the file to suit your needs.
 
-If you ever need help: https://torsion.org/borgmatic/#issues'''
+If you ever need help: https://torsion.org/borgmatic/#issues''',
     )

borgmatic/actions/config/validate.py

@@ -18,7 +18,7 @@ def run_validate(validate_arguments, configs):
     borgmatic.logger.add_custom_log_levels()
 
     if validate_arguments.show:
-        for config_path, config in configs.items():
+        for config in configs.values():
             if len(configs) > 1:
                 logger.answer('---')
 

borgmatic/actions/create.py

@@ -2,6 +2,9 @@ import logging
 
 import borgmatic.actions.json
 import borgmatic.borg.create
+import borgmatic.borg.feature
+import borgmatic.borg.rename
+import borgmatic.borg.repo_list
 import borgmatic.config.paths
 import borgmatic.config.validate
 import borgmatic.hooks.dispatch
@@ -28,18 +31,19 @@ def run_create(
     If create_arguments.json is True, yield the JSON output from creating the archive.
     '''
     if create_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, create_arguments.repository
+        repository,
+        create_arguments.repository,
     ):
         return
 
     if config.get('list_details') and config.get('progress'):
         raise ValueError(
-            'With the create action, only one of --list/--files/list_details and --progress/progress can be used.'
+            'With the create action, only one of --list/--files/list_details and --progress/progress can be used.',
         )
 
     if config.get('list_details') and create_arguments.json:
         raise ValueError(
-            'With the create action, only one of --list/--files/list_details and --json can be used.'
+            'With the create action, only one of --list/--files/list_details and --json can be used.',
         )
 
     logger.info(f'Creating archive{dry_run_label}')
@@ -53,7 +57,11 @@ def run_create(
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )
-        patterns = pattern.process_patterns(pattern.collect_patterns(config), working_directory)
+        patterns = pattern.process_patterns(
+            pattern.collect_patterns(config),
+            config,
+            working_directory,
+        )
         active_dumps = borgmatic.hooks.dispatch.call_hooks(
             'dump_data_sources',
             config,
@@ -68,10 +76,19 @@ def run_create(
         # we could end up with duplicate paths that cause Borg to hang when it tries to read from
         # the same named pipe twice.
         patterns = pattern.process_patterns(
-            patterns, working_directory, skip_expand_paths=config_paths
+            patterns,
+            config,
+            working_directory,
+            skip_expand_paths=config_paths,
         )
         stream_processes = [process for processes in active_dumps.values() for process in processes]
 
+        # If we have stream processes, we first create an archive with .checkpoint suffix. This is
+        # to make sure we only create a real archive if all the streaming processes completed
+        # successfully (create_archive will fail if a streaming process fails, but the archive might
+        # have already been created at this point).
+        use_checkpoint = bool(stream_processes)
+
         json_output = borgmatic.borg.create.create_archive(
             global_arguments.dry_run,
             repository['path'],
@@ -80,14 +97,41 @@ def run_create(
             local_borg_version,
             global_arguments,
             borgmatic_runtime_directory,
+            archive_suffix='.checkpoint' if use_checkpoint else '',
             local_path=local_path,
             remote_path=remote_path,
             json=create_arguments.json,
+            comment=create_arguments.comment,
             stream_processes=stream_processes,
         )
 
+        if use_checkpoint:
+            rename_checkpoint_archive(
+                repository['path'],
+                global_arguments,
+                config,
+                local_borg_version,
+                local_path,
+                remote_path,
+            )
+
         if json_output:
-            yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+            output = borgmatic.actions.json.parse_json(json_output, repository.get('label'))
+            if use_checkpoint:
+                # Patch archive name and ID
+                renamed_archive = borgmatic.borg.repo_list.get_latest_archive(
+                    repository['path'],
+                    config,
+                    local_borg_version,
+                    global_arguments,
+                    local_path,
+                    remote_path,
+                )
+
+                output['archive']['name'] = renamed_archive['name']
+                output['archive']['id'] = renamed_archive['id']
+
+            yield output
 
         borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
             'remove_data_source_dumps',
@@ -96,3 +140,60 @@ def run_create(
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )
+
+
+def rename_checkpoint_archive(
+    repository_path,
+    global_arguments,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    '''
+    Renames the latest archive to not have a '.checkpoint' suffix.
+
+    Raises ValueError if
+    - there is not latest archive
+    - the latest archive does not have a '.checkpoint' suffix
+
+    Implementation note: We cannot reliably get the just created archive name.
+    So we resort to listing the archives and picking the last one.
+
+    A similar comment applies to retrieving the ID of the renamed archive.
+    '''
+    archive = borgmatic.borg.repo_list.get_latest_archive(
+        repository_path,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path,
+        remote_path,
+        consider_checkpoints=True,
+    )
+
+    archive_name = archive['name']
+
+    if not archive_name.endswith('.checkpoint'):
+        raise ValueError(f'Latest archive did not have a .checkpoint suffix. Got: {archive_name}')
+
+    new_archive_name = archive_name.removesuffix('.checkpoint')
+
+    logger.info(f'Renaming archive {archive_name} -> {new_archive_name}')
+
+    borgmatic.borg.rename.rename_archive(
+        repository_path,
+        (
+            archive['id']
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.ARCHIVE_SERIES, local_borg_version
+            )
+            else archive['name']
+        ),
+        new_archive_name,
+        global_arguments.dry_run,
+        config,
+        local_borg_version,
+        local_path,
+        remote_path,
+    )

borgmatic/actions/delete.py

@@ -21,7 +21,8 @@ def run_delete(
     Run the "delete" action for the given repository and archive(s).
     '''
     if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, delete_arguments.repository
+        repository,
+        delete_arguments.repository,
     ):
         logger.answer('Deleting archives')
 

borgmatic/actions/export_key.py

@@ -19,7 +19,8 @@ def run_export_key(
     Run the "key export" action for the given repository.
     '''
     if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, export_arguments.repository
+        repository,
+        export_arguments.repository,
     ):
         logger.info('Exporting repository key')
         borgmatic.borg.export_key.export_key(

borgmatic/actions/export_tar.py

@@ -20,7 +20,8 @@ def run_export_tar(
     Run the "export-tar" action for the given repository.
     '''
     if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, export_tar_arguments.repository
+        repository,
+        export_tar_arguments.repository,
     ):
         logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
         borgmatic.borg.export_tar.export_tar_archive(

borgmatic/actions/extract.py

@@ -22,7 +22,8 @@ def run_extract(
     Run the "extract" action for the given repository.
     '''
     if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, extract_arguments.repository
+        repository,
+        extract_arguments.repository,
     ):
         logger.info(f'Extracting archive {extract_arguments.archive}')
         borgmatic.borg.extract.extract_archive(

borgmatic/actions/import_key.py

@@ -19,7 +19,8 @@ def run_import_key(
     Run the "key import" action for the given repository.
     '''
     if import_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, import_arguments.repository
+        repository,
+        import_arguments.repository,
     ):
         logger.info('Importing repository key')
         borgmatic.borg.import_key.import_key(

borgmatic/actions/info.py

@@ -24,10 +24,12 @@ def run_info(
     If info_arguments.json is True, yield the JSON output from the info for the archive.
     '''
     if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, info_arguments.repository
+        repository,
+        info_arguments.repository,
     ):
         if not info_arguments.json:
             logger.answer('Displaying archive summary information')
+
         archive_name = borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
             info_arguments.archive,

borgmatic/actions/list.py

@@ -23,7 +23,8 @@ def run_list(
     If list_arguments.json is True, yield the JSON output from listing the archive.
     '''
     if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, list_arguments.repository
+        repository,
+        list_arguments.repository,
     ):
         if not list_arguments.json:
             if list_arguments.find_paths:  # pragma: no cover

borgmatic/actions/mount.py

@@ -20,7 +20,8 @@ def run_mount(
     Run the "mount" action for the given repository.
     '''
     if mount_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, mount_arguments.repository
+        repository,
+        mount_arguments.repository,
     ):
         if mount_arguments.archive:
             logger.info(f'Mounting archive {mount_arguments.archive}')

borgmatic/actions/pattern.py

@@ -47,7 +47,8 @@ def collect_patterns(config):
         return (
             tuple(
                 borgmatic.borg.pattern.Pattern(
-                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
+                    source_directory,
+                    source=borgmatic.borg.pattern.Pattern_source.CONFIG,
                 )
                 for source_directory in config.get('source_directories', ())
             )
@@ -67,7 +68,7 @@ def collect_patterns(config):
             + tuple(
                 parse_pattern(pattern_line.strip())
                 for filename in config.get('patterns_from', ())
-                for pattern_line in open(filename).readlines()
+                for pattern_line in open(filename, encoding='utf-8').readlines()
                 if not pattern_line.lstrip().startswith('#')
                 if pattern_line.strip()
             )
@@ -77,7 +78,7 @@ def collect_patterns(config):
                     borgmatic.borg.pattern.Pattern_style.FNMATCH,
                 )
                 for filename in config.get('exclude_from', ())
-                for exclude_line in open(filename).readlines()
+                for exclude_line in open(filename, encoding='utf-8').readlines()
                 if not exclude_line.lstrip().startswith('#')
                 if exclude_line.strip()
             )
@@ -112,9 +113,8 @@ def expand_directory(directory, working_directory):
             glob_path
             # If these are equal, that means we didn't add any working directory prefix above.
             if normalized_directory == expanded_directory
-            # Remove the working directory prefix that we added above in order to make glob() work.
+            # Remove the working directory prefix added above in order to make glob() work. We
-            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
+            # can't use os.path.relpath() here because it collapses any use of Borg's slashdot hack.
-            # hack.
             else glob_path.removeprefix(working_directory_prefix)
         )
         for glob_path in glob_paths
@@ -161,10 +161,35 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None):
                 )
             )
             for pattern in patterns
-        )
+        ),
     )
 
 
+def get_existent_path_or_parent(path):
+    '''
+    Given a path, return it if it exists. Otherwise, return the longest parent directory of the path
+    that exists. Return None if none of these paths exist.
+
+    This is used below for finding an existent path prefix of pattern's path, which is necessary if
+    the path contain globs or other special characters that we don't want to try to interpret
+    (because we want to leave that responsibility to Borg).
+    '''
+    if path.startswith('/e2e/'):
+        return None
+
+    try:
+        return next(
+            candidate_path
+            for candidate_path in (
+                path,
+                *tuple(str(parent) for parent in pathlib.PurePath(path).parents),
+            )
+            if os.path.exists(candidate_path)
+        )
+    except StopIteration:
+        return None
+
+
 def device_map_patterns(patterns, working_directory=None):
     '''
     Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
@@ -174,36 +199,44 @@ def device_map_patterns(patterns, working_directory=None):
 
     This is handy for determining whether two different pattern paths are on the same filesystem
     (have the same device identifier).
+
+    This function only considers the start of a pattern's path—from the start of the path up until
+    there's a path component with a glob or other non-literal character. If there are no such
+    characters, the whole path is considered. The rationale is that it's not feasible for borgmatic
+    to interpret Borg's patterns to see which actual files (and therefore devices) they map to. So
+    for instance, a pattern with a path of "/var/log/*/data" would end up with its device set to the
+    device of "/var/log"—ignoring the "/*/data" part due to that glob.
+
+    The one exception is that if a regular expression pattern path starts with "^", that will get
+    stripped off for purposes of determining its device.
     '''
     return tuple(
         borgmatic.borg.pattern.Pattern(
             pattern.path,
             pattern.type,
             pattern.style,
-            device=pattern.device
+            device=pattern.device or (os.stat(existent_path).st_dev if existent_path else None),
-            or (
-                os.stat(full_path).st_dev
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and os.path.exists(full_path)
-                else None
-            ),
             source=pattern.source,
         )
         for pattern in patterns
-        for full_path in (os.path.join(working_directory or '', pattern.path),)
+        for existent_path in (
+            get_existent_path_or_parent(
+                os.path.join(working_directory or '', pattern.path.lstrip('^')),
+            ),
+        )
     )
 
 
-def deduplicate_patterns(patterns):
+def deduplicate_patterns(patterns, config):
     '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and a configuration dict, return
-    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
+    them with all duplicate root child patterns removed. For instance, if two root patterns are
-    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
+    given with paths "/foo" and "/foo/bar", return just the one with "/foo". Non-root patterns are
-    modification.
+    passed through without modification.
 
-    The one exception to deduplication is two paths are on different filesystems (devices). In that
+    The one exception to deduplication is if two paths are on different filesystems (devices) and
-    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
+    "one_file_system" is True in the given configuration. In that case, the paths won't get
-    one_file_system option is true).
+    deduplicated, because Borg won't cross filesystem boundaries when "one_file_system" is True.
 
     The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
     child patterns, because it will naturally spider the contents of the parent pattern's path. And
@@ -221,7 +254,8 @@ def deduplicate_patterns(patterns):
         parents = pathlib.PurePath(pattern.path).parents
 
         # If another directory in the given list is a parent of current directory (even n levels up)
-        # and both are on the same filesystem, then the current directory is a duplicate.
+        # and both are on the same filesystem (or one_file_system is not set), then the current
+        # directory is a duplicate.
         for other_pattern in patterns:
             if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
                 continue
@@ -229,7 +263,10 @@ def deduplicate_patterns(patterns):
             if any(
                 pathlib.PurePath(other_pattern.path) == parent
                 and pattern.device is not None
-                and other_pattern.device == pattern.device
+                and (
+                    other_pattern.device == pattern.device
+                    or config.get('one_file_system') is not True
+                )
                 for parent in parents
             ):
                 break
@@ -239,12 +276,11 @@ def deduplicate_patterns(patterns):
     return tuple(deduplicated.keys())
 
 
-def process_patterns(patterns, working_directory, skip_expand_paths=None):
+def process_patterns(patterns, config, working_directory, skip_expand_paths=None):
     '''
-    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
+    Given a sequence of Borg patterns, a configuration dict, a configured working directory, and a
-    "root" patterns, returning the resulting root and non-root patterns as a list.
+    sequence of paths to skip path expansion for, expand and deduplicate any "root" patterns,
-
+    returning the resulting root and non-root patterns as a list.
-    If any paths are given to skip, don't expand them.
     '''
     skip_paths = set(skip_expand_paths or ())
 
@@ -255,7 +291,8 @@ def process_patterns(patterns, working_directory, skip_expand_paths=None):
                     patterns,
                     working_directory=working_directory,
                     skip_paths=skip_paths,
-                )
+                ),
-            )
+            ),
-        )
+            config,
+        ),
     )

borgmatic/actions/prune.py

@@ -22,7 +22,8 @@ def run_prune(
     Run the "prune" action for the given repository.
     '''
     if prune_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, prune_arguments.repository
+        repository,
+        prune_arguments.repository,
     ):
         return
 

borgmatic/actions/recreate.py

@@ -26,7 +26,8 @@ def run_recreate(
     Run the "recreate" action for the given repository.
     '''
     if recreate_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, recreate_arguments.repository
+        repository,
+        recreate_arguments.repository,
     ):
         if recreate_arguments.archive:
             logger.answer(f'Recreating archive {recreate_arguments.archive}')
@@ -35,7 +36,9 @@ def run_recreate(
 
         # Collect and process patterns.
         processed_patterns = process_patterns(
-            collect_patterns(config), borgmatic.config.paths.get_working_directory(config)
+            collect_patterns(config),
+            config,
+            borgmatic.config.paths.get_working_directory(config),
         )
 
         archive = borgmatic.borg.repo_list.resolve_archive_name(
@@ -51,13 +54,13 @@ def run_recreate(
         if archive and archive.endswith('.recreate'):
             if recreate_arguments.archive == 'latest':
                 raise ValueError(
-                    f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.'
+                    f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.',
-                )
-            else:
-                raise ValueError(
-                    f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.'
                 )
 
+            raise ValueError(
+                f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.',
+            )
+
         try:
             borgmatic.borg.recreate.recreate_archive(
                 repository['path'],
@@ -74,11 +77,12 @@ def run_recreate(
             if error.returncode == BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS:
                 if recreate_arguments.target:
                     raise ValueError(
-                        f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.'
+                        f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.',
                     )
-                elif archive:
+
+                if archive:
                     raise ValueError(
-                        f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.'
+                        f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.',
                     )
 
             raise

borgmatic/actions/repo_create.py

@@ -19,7 +19,8 @@ def run_repo_create(
     Run the "repo-create" action for the given repository.
     '''
     if repo_create_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, repo_create_arguments.repository
+        repository,
+        repo_create_arguments.repository,
     ):
         return
 
@@ -29,7 +30,7 @@ def run_repo_create(
 
     if not encryption_mode:
         raise ValueError(
-            'With the repo-create action, either the --encryption flag or the repository encryption option is required.'
+            'With the repo-create action, either the --encryption flag or the repository encryption option is required.',
         )
 
     borgmatic.borg.repo_create.create_repository(

borgmatic/actions/repo_delete.py

@@ -18,10 +18,11 @@ def run_repo_delete(
     Run the "repo-delete" action for the given repository.
     '''
     if repo_delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_delete_arguments.repository
+        repository,
+        repo_delete_arguments.repository,
     ):
         logger.answer(
-            'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '')
+            'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else ''),
         )
 
         borgmatic.borg.repo_delete.delete_repository(

borgmatic/actions/repo_info.py

@@ -22,7 +22,8 @@ def run_repo_info(
     If repo_info_arguments.json is True, yield the JSON output from the info for the repository.
     '''
     if repo_info_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_info_arguments.repository
+        repository,
+        repo_info_arguments.repository,
     ):
         if not repo_info_arguments.json:
             logger.answer('Displaying repository summary information')

borgmatic/actions/repo_list.py

@@ -22,7 +22,8 @@ def run_repo_list(
     If repo_list_arguments.json is True, yield the JSON output from listing the repository.
     '''
     if repo_list_arguments.repository is None or borgmatic.config.validate.repositories_match(
-        repository, repo_list_arguments.repository
+        repository,
+        repo_list_arguments.repository,
     ):
         if not repo_list_arguments.json:
             logger.answer('Listing repository')

borgmatic/actions/restore.py

@@ -22,8 +22,8 @@ UNSPECIFIED = object()
 
 Dump = collections.namedtuple(
     'Dump',
-    ('hook_name', 'data_source_name', 'hostname', 'port'),
+    ('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'),
-    defaults=('localhost', None),
+    defaults=(None, None, None, None),
 )
 
 
@@ -33,17 +33,24 @@ def dumps_match(first, second, default_port=None):
     indicates that the field should match any value. If a default port is given, then consider any
     dump having that port to match with a dump having a None port.
     '''
-    for field_name in first._fields:
+    # label kinda counts as an unique id, if they match ignore host/container/port
+    if first.label not in {None, UNSPECIFIED} and first.label == second.label:
+        field_list = ('hook_name', 'data_source_name')
+    else:
+        field_list = Dump._fields
+
+    for field_name in field_list:
         first_value = getattr(first, field_name)
         second_value = getattr(second, field_name)
 
         if default_port is not None and field_name == 'port':
             if first_value == default_port and second_value is None:
                 continue
+
             if second_value == default_port and first_value is None:
                 continue
 
-        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
+        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:  # noqa: PLR1714
             continue
 
         if first_value != second_value:
@@ -56,16 +63,19 @@ def render_dump_metadata(dump):
     '''
     Given a Dump instance, make a display string describing it for use in log messages.
     '''
+    label = dump.label or UNSPECIFIED
     name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
-    hostname = dump.hostname or UNSPECIFIED
+    host = dump.container or dump.hostname or UNSPECIFIED
     port = None if dump.port is UNSPECIFIED else dump.port
 
-    if port:
+    if label is not UNSPECIFIED:
-        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
+        metadata = f'{name}@{label}'
+    elif port:
+        metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}'
     else:
-        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
+        metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}'
 
-    if dump.hook_name not in (None, UNSPECIFIED):
+    if dump.hook_name not in {None, UNSPECIFIED}:
         return f'{metadata} ({dump.hook_name})'
 
     return metadata
@@ -98,8 +108,10 @@ def get_configured_data_source(config, restore_dump):
             Dump(
                 hook_name,
                 hook_data_source.get('name'),
-                hook_data_source.get('hostname', 'localhost'),
+                hook_data_source.get('hostname'),
                 hook_data_source.get('port'),
+                hook_data_source.get('label') or UNSPECIFIED,
+                hook_data_source.get('container'),
             ),
             restore_dump,
             default_port,
@@ -111,14 +123,15 @@ def get_configured_data_source(config, restore_dump):
 
     if len(matching_dumps) > 1:
         raise ValueError(
-            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
+            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured',
         )
 
     return matching_dumps[0]
 
 
 def strip_path_prefix_from_extracted_dump_destination(
-    destination_path, borgmatic_runtime_directory
+    destination_path,
+    borgmatic_runtime_directory,
 ):
     '''
     Directory-format dump files get extracted into a temporary directory containing a path prefix
@@ -145,7 +158,8 @@ def strip_path_prefix_from_extracted_dump_destination(
             continue
 
         shutil.move(
-            subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)
+            subdirectory_path,
+            os.path.join(borgmatic_runtime_directory, databases_directory),
         )
         break
 
@@ -169,7 +183,14 @@ def restore_single_dump(
     that data source from the archive.
     '''
     dump_metadata = render_dump_metadata(
-        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
+        Dump(
+            hook_name,
+            data_source['name'],
+            data_source.get('hostname'),
+            data_source.get('port'),
+            data_source.get('label') or UNSPECIFIED,
+            data_source.get('container'),
+        ),
     )
 
     logger.info(f'Restoring data source {dump_metadata}')
@@ -197,8 +218,8 @@ def restore_single_dump(
             archive=archive_name,
             paths=[
                 borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
-                    dump_patterns
+                    dump_patterns,
-                )
+                ),
             ],
             config=config,
             local_borg_version=local_borg_version,
@@ -213,7 +234,8 @@ def restore_single_dump(
 
         if destination_path and not global_arguments.dry_run:
             strip_path_prefix_from_extracted_dump_destination(
-                destination_path, borgmatic_runtime_directory
+                destination_path,
+                borgmatic_runtime_directory,
             )
     finally:
         if destination_path and not global_arguments.dry_run:
@@ -248,8 +270,68 @@ def collect_dumps_from_archive(
     borgmatic runtime directory, query the archive for the names of data sources dumps it contains
     and return them as a set of Dump instances.
     '''
+    dumps_from_archive = set()
+
+    # There is (at most) one dump metadata file per data source hook. Load each.
+    for dumps_metadata_path in borgmatic.borg.list.capture_archive_listing(
+        repository,
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        list_paths=[
+            'sh:'
+            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
+                base_directory,
+                '*_databases/dumps.json',
+            )
+            # Probe for dump metadata files in multiple locations, as the default location is
+            # "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the
+            # borgmatic runtime directory for older versions of Borg.
+            for base_directory in (
+                'borgmatic',
+                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+            )
+        ],
+        local_path=local_path,
+        remote_path=remote_path,
+    ):
+        if not dumps_metadata_path:
+            continue
+
+        dumps_from_archive.update(
+            set(
+                borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata(
+                    borgmatic.borg.extract.extract_archive(
+                        global_arguments.dry_run,
+                        repository,
+                        archive,
+                        [dumps_metadata_path],
+                        config,
+                        local_borg_version,
+                        global_arguments,
+                        local_path=local_path,
+                        remote_path=remote_path,
+                        extract_to_stdout=True,
+                    )
+                    .stdout.read()
+                    .decode(),
+                    dumps_metadata_path,
+                )
+            )
+        )
+
+    # If we've successfully loaded any dumps metadata, we're done.
+    if dumps_from_archive:
+        logger.debug('Collecting database dumps from archive data source dumps metadata files')
+
+        return dumps_from_archive
+
+    # No dumps metadata files were found, so for backwards compatibility, fall back to parsing the
+    # paths of dumps found in the archive to get their respective dump metadata.
+    logger.debug('Collecting database dumps from archive data source dump paths (fallback)')
     borgmatic_source_directory = str(
-        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
+        pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)),
     )
 
     # Probe for the data source dumps in multiple locations, as the default location has moved to
@@ -264,7 +346,8 @@ def collect_dumps_from_archive(
         list_paths=[
             'sh:'
             + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
-                base_directory, '*_databases/*/*'
+                base_directory,
+                '*_databases/*/*',
             )
             for base_directory in (
                 'borgmatic',
@@ -276,9 +359,6 @@ def collect_dumps_from_archive(
         remote_path=remote_path,
     )
 
-    # Parse the paths of dumps found in the archive to get their respective dump metadata.
-    dumps_from_archive = set()
-
     for dump_path in dump_paths:
         if not dump_path:
             continue
@@ -291,7 +371,8 @@ def collect_dumps_from_archive(
         ):
             try:
                 (hook_name, host_and_port, data_source_name) = dump_path.split(
-                    base_directory + os.path.sep, 1
+                    base_directory + os.path.sep,
+                    1,
                 )[1].split(os.path.sep)[0:3]
             except (ValueError, IndexError):
                 continue
@@ -308,13 +389,17 @@ def collect_dumps_from_archive(
             except (ValueError, TypeError):
                 port = None
 
-            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
+            dumps_from_archive.add(
+                Dump(
+                    hook_name, data_source_name, None if hostname == 'localhost' else hostname, port
+                )
+            )
 
             # We've successfully parsed the dump path, so need to probe any further.
             break
         else:
             logger.warning(
-                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
+                f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}',
             )
 
     return dumps_from_archive
@@ -345,6 +430,8 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
                 data_source_name=name,
                 hostname=restore_arguments.original_hostname or UNSPECIFIED,
                 port=restore_arguments.original_port,
+                label=restore_arguments.original_label or UNSPECIFIED,
+                container=restore_arguments.original_container or UNSPECIFIED,
             )
             for name in restore_arguments.data_sources or (UNSPECIFIED,)
         }
@@ -352,13 +439,17 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
         or restore_arguments.data_sources
         or restore_arguments.original_hostname
         or restore_arguments.original_port
+        or restore_arguments.original_label
+        or restore_arguments.original_container
         else {
             Dump(
                 hook_name=UNSPECIFIED,
                 data_source_name='all',
                 hostname=UNSPECIFIED,
                 port=UNSPECIFIED,
-            )
+                label=UNSPECIFIED,
+                container=UNSPECIFIED,
+            ),
         }
     )
     missing_dumps = set()
@@ -385,7 +476,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
             dumps_to_restore.add(matching_dumps[0])
         else:
             raise ValueError(
-                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
+                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.',
             )
 
     if missing_dumps:
@@ -394,7 +485,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
         )
 
         raise ValueError(
-            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
+            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive",
         )
 
     return dumps_to_restore
@@ -410,14 +501,15 @@ def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
         raise ValueError('No data source dumps were found to restore')
 
     missing_dumps = sorted(
-        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
+        dumps_to_restore - dumps_actually_restored,
+        key=lambda dump: dump.data_source_name,
     )
 
     if missing_dumps:
         rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
 
         raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
+            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration",
         )
 
 
@@ -438,7 +530,8 @@ def run_restore(
     matching dump in the archive.
     '''
     if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
-        repository, restore_arguments.repository
+        repository,
+        restore_arguments.repository,
     ):
         return
 
@@ -476,6 +569,7 @@ def run_restore(
 
         dumps_actually_restored = set()
         connection_params = {
+            'container': restore_arguments.container,
             'hostname': restore_arguments.hostname,
             'port': restore_arguments.port,
             'username': restore_arguments.username,
@@ -489,13 +583,19 @@ def run_restore(
                 config,
                 restore_dump,
             )
-
             # For a dump that wasn't found via an exact match in the configuration, try to fallback
             # to an "all" data source.
             if not found_data_source:
                 found_data_source = get_configured_data_source(
                     config,
-                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
+                    Dump(
+                        restore_dump.hook_name,
+                        'all',
+                        restore_dump.hostname,
+                        restore_dump.port,
+                        restore_dump.label,
+                        restore_dump.container,
+                    ),
                 )
 
                 if not found_data_source:
@@ -515,7 +615,7 @@ def run_restore(
                 remote_path,
                 archive_name,
                 restore_dump.hook_name,
-                dict(found_data_source, **{'schemas': restore_arguments.schemas}),
+                dict(found_data_source, schemas=restore_arguments.schemas),
                 connection_params,
                 borgmatic_runtime_directory,
             )

borgmatic/actions/transfer.py

@@ -19,7 +19,7 @@ def run_transfer(
     '''
     if transfer_arguments.archive and config.get('match_archives'):
         raise ValueError(
-            'With the transfer action, only one of --archive and --match-archives/match_archives can be used.'
+            'With the transfer action, only one of --archive and --match-archives/match_archives can be used.',
         )
 
     logger.info('Transferring archives to repository')

borgmatic/borg/borg.py

@@ -39,9 +39,9 @@ def run_arbitrary_borg(
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
 
-        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
+        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES:
             logger.warning(
-                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
+                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}",
             )
     except IndexError:
         borg_command = ()
@@ -57,16 +57,14 @@ def run_arbitrary_borg(
         + command_options
     )
 
-    return execute_command(
+    return execute_command(  # noqa: S604
         tuple(shlex.quote(part) for part in full_command),
         output_file=DO_NOT_CAPTURE,
         shell=True,
         environment=dict(
             (environment.make_environment(config) or {}),
-            **{
+            BORG_REPO=repository_path,
-                'BORG_REPO': repository_path,
+            ARCHIVE=archive if archive else '',
-                'ARCHIVE': archive if archive else '',
-            },
         ),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,

borgmatic/borg/change_passphrase.py

@@ -49,7 +49,7 @@ def change_passphrase(
     config_without_passphrase = {
         option_name: value
         for (option_name, value) in config.items()
-        if option_name not in ('encryption_passphrase', 'encryption_passcommand')
+        if option_name not in {'encryption_passphrase', 'encryption_passcommand'}
     }
 
     borgmatic.execute.execute_command(
@@ -63,5 +63,5 @@ def change_passphrase(
     )
 
     logger.answer(
-        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)"
+        f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)",
     )

borgmatic/borg/check.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import logging
+import shlex
 
 import borgmatic.config.paths
 from borgmatic.borg import environment, feature, flags, repo_info
@@ -41,11 +42,12 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument
 
     if check_last:
         logger.warning(
-            'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
+            'Ignoring check_last option, as "archives" or "data" are not in consistency checks',
         )
+
     if prefix:
         logger.warning(
-            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
+            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks',
         )
 
     return ()
@@ -75,13 +77,18 @@ def make_check_name_flags(checks, archive_filter_flags):
         return common_flags
 
     return (
-        tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives'))
+        tuple(f'--{check}-only' for check in checks if check in {'repository', 'archives'})
         + common_flags
     )
 
 
 def get_repository_id(
-    repository_path, config, local_borg_version, global_arguments, local_path, remote_path
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Given a local or remote repository path, a configuration dict, the local Borg version, global
@@ -99,7 +106,7 @@ def get_repository_id(
                 global_arguments,
                 local_path,
                 remote_path,
-            )
+            ),
         )['repository']['id']
     except (json.JSONDecodeError, KeyError):
         raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
@@ -126,8 +133,10 @@ def check_archives(
     extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
 
     verbosity_flags = ()
+
     if logger.isEnabledFor(logging.INFO):
         verbosity_flags = ('--info',)
+
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
 
@@ -171,7 +180,7 @@ def check_archives(
             + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
             + (('--progress',) if config.get('progress') else ())
-            + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+            + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
             + flags.make_repository_flags(repository_path, local_borg_version)
         )
 

borgmatic/borg/compact.py

@@ -1,7 +1,8 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
-from borgmatic.borg import environment, flags
+from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
@@ -37,12 +38,17 @@ def compact_segments(
         + (('--threshold', str(threshold)) if threshold else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (
+            ('--dry-run',)
+            if dry_run and feature.available(feature.Feature.DRY_RUN_COMPACT, local_borg_version)
+            else ()
+        )
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    if dry_run:
+    if dry_run and not feature.available(feature.Feature.DRY_RUN_COMPACT, local_borg_version):
-        logging.info('Skipping compact (dry run)')
+        logger.info('Skipping compact (dry run)')
         return
 
     execute_command(

borgmatic/borg/create.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import pathlib
+import shlex
 import stat
 import textwrap
 
@@ -44,35 +45,38 @@ def any_parent_directories(path, candidate_parents):
     return False
 
 
-def collect_special_file_paths(
+def validate_planned_backup_paths(
     dry_run,
     create_command,
     config,
+    patterns,
     local_path,
     working_directory,
     borgmatic_runtime_directory,
 ):
     '''
     Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
-    a working directory, and the borgmatic runtime directory, collect the paths for any special
+    a working directory, and the borgmatic runtime directory, perform a "borg create --dry-run" to
-    files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
+    determine whether Borg's planned paths to include in a backup look good. Specifically, if the
-    during a create. These are all paths that could cause Borg to hang if its --read-special flag is
+    given runtime directory exists, validate that it will be included in a backup and hasn't been
-    used.
+    excluded.
 
-    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
+    Raise ValueError if the runtime directory has been excluded via "exclude_patterns" or similar,
-    its own special files there for database dumps and we don't want those omitted.
+    because any features that rely on the runtime directory getting backed up will break.  For
-
+    instance, without the runtime directory, Borg can't consume any database dumps and borgmatic may
-    Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
+    hang waiting for them to be consumed.
-    plans to backup, that means the user must have excluded the runtime directory (e.g. via
-    "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
-    consume any database dumps and therefore borgmatic will hang when it tries to do so.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
     # files including any named pipe we've created. And omit "--filter" because that can break the
-    # paths output parsing below such that path lines no longer start with th expected "- ".
+    # paths output parsing below such that path lines no longer start with the expected "- ".
     paths_output = execute_command_and_capture_output(
-        flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
+        (
-        + ('--dry-run', '--list'),
+            *flags.omit_flag_and_value(
+                flags.omit_flag(create_command, '--exclude-nodump'), '--filter'
+            ),
+            '--dry-run',
+            '--list',
+        ),
         capture_stderr=True,
         working_directory=working_directory,
         environment=environment.make_environment(config),
@@ -85,30 +89,36 @@ def collect_special_file_paths(
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.split('\n')
-        if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
+        if path_line and path_line.startswith(('- ', '+ '))
     )
 
-    # These are the subset of those files that contain the borgmatic runtime directory.
+    # These are the subset of output paths contained within the borgmatic runtime directory.
-    paths_containing_runtime_directory = {}
+    paths_inside_runtime_directory = {
+        path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
+    }
 
-    if os.path.exists(borgmatic_runtime_directory):
+    # If the runtime directory isn't present in the source patterns, then we shouldn't expect it to
-        paths_containing_runtime_directory = {
+    # be in the paths output from the Borg dry run.
-            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
+    runtime_directory_present_in_patterns = any(
-        }
+        pattern
-
+        for pattern in patterns
-        # If no paths to backup contain the runtime directory, it must've been excluded.
+        if any_parent_directories(pattern.path, (borgmatic_runtime_directory,))
-        if not paths_containing_runtime_directory and not dry_run:
+        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-            raise ValueError(
-                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
-            )
-
-    return tuple(
-        path
-        for path in paths
-        if special_file(path, working_directory)
-        if path not in paths_containing_runtime_directory
     )
 
+    # If no paths to backup are inside the runtime directory, it must've been excluded.
+    if (
+        not paths_inside_runtime_directory
+        and runtime_directory_present_in_patterns
+        and not dry_run
+        and os.path.exists(borgmatic_runtime_directory)
+    ):
+        raise ValueError(
+            f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.',
+        )
+
+    return tuple(path for path in paths if path not in paths_inside_runtime_directory)
+
 
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
 
@@ -121,23 +131,28 @@ def make_base_create_command(
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
+    archive_suffix='',
     local_path='borg',
     remote_path=None,
     json=False,
+    comment=None,
     stream_processes=None,
 ):
     '''
     Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version,
+    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version, global
-    global arguments as an argparse.Namespace instance, and a sequence of borgmatic source
+    arguments as an argparse.Namespace instance, the borgmatic runtime directory, a string suffix to
-    directories, return a tuple of (base Borg create command flags, Borg create command positional
+    add to the archive name, the local Borg path, the remote Borg path, whether to output JSON,
-    arguments, open pattern file handle).
+    comment text to add to the created archive, and a sequence of processes streaming data to Borg,
+    return a tuple of (base Borg create command flags, Borg create command positional arguments,
+    open pattern file handle).
     '''
     if config.get('source_directories_must_exist', False):
         borgmatic.borg.pattern.check_all_root_patterns_exist(patterns)
 
     patterns_file = borgmatic.borg.pattern.write_patterns_file(
-        patterns, borgmatic_runtime_directory
+        patterns,
+        borgmatic_runtime_directory,
     )
     checkpoint_interval = config.get('checkpoint_interval', None)
     checkpoint_volume = config.get('checkpoint_volume', None)
@@ -149,8 +164,9 @@ def make_base_create_command(
     lock_wait = config.get('lock_wait', None)
     list_filter_flags = flags.make_list_filter_flags(local_borg_version, dry_run)
     files_cache = config.get('files_cache')
-    archive_name_format = config.get(
+    archive_name_format = (
-        'archive_name_format', flags.get_default_archive_name_format(local_borg_version)
+        config.get('archive_name_format', flags.get_default_archive_name_format(local_borg_version))
+        + archive_suffix
     )
     extra_borg_options = config.get('extra_borg_options', {}).get('create', '')
 
@@ -183,6 +199,7 @@ def make_base_create_command(
         + ('create',)
         + (('--patterns-from', patterns_file.name) if patterns_file else ())
         + flags.make_exclude_flags(config)
+        + (('--comment', comment) if comment else ())
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
@@ -207,29 +224,36 @@ def make_base_create_command(
             else ()
         )
         + (('--dry-run',) if dry_run else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
     )
 
     create_positional_arguments = flags.make_repository_archive_flags(
-        repository_path, archive_name_format, local_borg_version
+        repository_path,
+        archive_name_format,
+        local_borg_version,
+    )
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    logger.debug('Checking file paths Borg plans to include')
+    planned_backup_paths = validate_planned_backup_paths(
+        dry_run,
+        create_flags + create_positional_arguments,
+        config,
+        patterns,
+        local_path,
+        working_directory,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
     )
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
     if stream_processes and not config.get('read_special'):
         logger.warning(
-            'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
+            'Ignoring configured "read_special" value of false, as true is needed for database hooks.',
         )
-        working_directory = borgmatic.config.paths.get_working_directory(config)
 
-        logger.debug('Collecting special file paths')
+        special_file_paths = tuple(
-        special_file_paths = collect_special_file_paths(
+            path for path in planned_backup_paths if special_file(path, working_directory)
-            dry_run,
-            create_flags + create_positional_arguments,
-            config,
-            local_path,
-            working_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
         )
 
         if special_file_paths:
@@ -239,7 +263,7 @@ def make_base_create_command(
                 placeholder=' ...',
             )
             logger.warning(
-                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
+                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
             )
             patterns_file = borgmatic.borg.pattern.write_patterns_file(
                 tuple(
@@ -269,15 +293,20 @@ def create_archive(
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
+    archive_suffix='',
     local_path='borg',
     remote_path=None,
     json=False,
+    comment=None,
     stream_processes=None,
 ):
     '''
     Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of loaded configuration paths, the local Borg version, and global arguments as an
+    sequence of loaded configuration paths, the local Borg version, global arguments as an
-    argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
+    argparse.Namespace instance, the borgmatic runtime directory, a string suffix to add to the
+    archive name, the local Borg path, the remote Borg path, whether to output JSON, and comment
+    text to add to the created archive, and a sequence of processes streaming data to Borg, create a
+    Borg archive and return Borg's JSON output (if any).
 
     If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
     create command while also triggering the given processes to produce output.
@@ -286,7 +315,7 @@ def create_archive(
 
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
+    (create_flags, create_positional_arguments, _) = make_base_create_command(
         dry_run,
         repository_path,
         config,
@@ -294,9 +323,11 @@ def create_archive(
         local_borg_version,
         global_arguments,
         borgmatic_runtime_directory,
+        archive_suffix,
         local_path,
         remote_path,
         json,
+        comment,
         stream_processes,
     )
 
@@ -331,7 +362,8 @@ def create_archive(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
-    elif output_log_level is None:
+
+    if output_log_level is None:
         return execute_command_and_capture_output(
             create_flags + create_positional_arguments,
             working_directory=working_directory,
@@ -339,13 +371,15 @@ def create_archive(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
-    else:
+
-        execute_command(
+    execute_command(
-            create_flags + create_positional_arguments,
+        create_flags + create_positional_arguments,
-            output_log_level,
+        output_log_level,
-            output_file,
+        output_file,
-            working_directory=working_directory,
+        working_directory=working_directory,
-            environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
-            borg_local_path=local_path,
+        borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+        borg_exit_codes=borg_exit_codes,
-        )
+    )
+
+    return None

borgmatic/borg/delete.py

@@ -11,6 +11,9 @@ import borgmatic.execute
 logger = logging.getLogger(__name__)
 
 
+FORCE_HARDER_FLAG_COUNT = 2
+
+
 def make_delete_command(
     repository,
     config,
@@ -36,7 +39,10 @@ def make_delete_command(
         + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
         + borgmatic.borg.flags.make_flags('list', config.get('list_details'))
         + (
-            (('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
+            (
+                ('--force',)
+                + (('--force',) if delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
+            )
             if delete_arguments.force
             else ()
         )
@@ -98,10 +104,11 @@ def delete_archives(
         for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
     ):
         if borgmatic.borg.feature.available(
-            borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+            borgmatic.borg.feature.Feature.REPO_DELETE,
+            local_borg_version,
         ):
             logger.warning(
-                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.'
+                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.',
             )
 
         repo_delete_arguments = argparse.Namespace(

borgmatic/borg/environment.py

@@ -14,13 +14,16 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'temporary_directory': 'TMPDIR',
 }
 
-DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = {
+DEFAULT_BOOL_OPTION_TO_UNCONDITIONAL_ENVIRONMENT_VARIABLE = {
-    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
-    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
 }
 
-DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
-    'check_i_know_what_i_am_doing': 'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING',
+    'debug_passphrase': 'BORG_DEBUG_PASSPHRASE',
+    'display_passphrase': 'BORG_DISPLAY_PASSPHRASE',
+    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
+    'use_chunks_archive': 'BORG_USE_CHUNKS_ARCHIVE',
 }
 
 
@@ -61,7 +64,8 @@ def make_environment(config):
         environment.pop('BORG_PASSCOMMAND', None)
 
     passphrase = borgmatic.hooks.credential.parse.resolve_credential(
-        config.get('encryption_passphrase'), config
+        config.get('encryption_passphrase'),
+        config,
     )
 
     if passphrase is None:
@@ -82,15 +86,15 @@ def make_environment(config):
     for (
         option_name,
         environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
         if os.environ.get(environment_variable_name) is None:
             value = config.get(option_name)
-            environment[environment_variable_name] = 'yes' if value else 'no'
+            environment[environment_variable_name] = 'YES' if value else 'NO'
 
     for (
         option_name,
         environment_variable_name,
-    ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items():
+    ) in DEFAULT_BOOL_OPTION_TO_UNCONDITIONAL_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
         if value is not None:
             environment[environment_variable_name] = 'YES' if value else 'NO'

borgmatic/borg/export_key.py

@@ -35,7 +35,7 @@ def export_key(
     if export_arguments.path and export_arguments.path != '-':
         if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
             raise FileExistsError(
-                f'Destination path {export_arguments.path} already exists. Aborting.'
+                f'Destination path {export_arguments.path} already exists. Aborting.',
             )
 
         output_file = None

borgmatic/borg/export_tar.py

@@ -56,13 +56,10 @@ def export_tar_archive(
         + (tuple(paths) if paths else ())
     )
 
-    if config.get('list_details'):
+    output_log_level = logging.ANSWER if config.get('list_details') else logging.INFO
-        output_log_level = logging.ANSWER
-    else:
-        output_log_level = logging.INFO
 
     if dry_run:
-        logging.info('Skipping export to tar file (dry run)')
+        logger.info('Skipping export to tar file (dry run)')
         return
 
     execute_command(

borgmatic/borg/extract.py

@@ -52,7 +52,9 @@ def extract_last_archive_dry_run(
         + verbosity_flags
         + list_flag
         + flags.make_repository_archive_flags(
-            repository_path, last_archive_name, local_borg_version
+            repository_path,
+            last_archive_name,
+            local_borg_version,
         )
     )
 
@@ -178,3 +180,5 @@ def extract_archive(
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
+
+    return None

borgmatic/borg/feature.py

@@ -18,6 +18,7 @@ class Feature(Enum):
     EXCLUDED_FILES_MINUS = 12
     ARCHIVE_SERIES = 13
     NO_PRUNE_STATS = 14
+    DRY_RUN_COMPACT = 15
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
@@ -35,6 +36,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
     Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
     Feature.ARCHIVE_SERIES: parse('2.0.0b11'),  # identically named archives form a series
     Feature.NO_PRUNE_STATS: parse('2.0.0b10'),  # prune --stats is not available
+    Feature.DRY_RUN_COMPACT: parse('1.2.9'),  # borg compact --dry-run support
 }
 
 

borgmatic/borg/flags.py

@@ -34,7 +34,7 @@ def make_flags_from_arguments(arguments, excludes=()):
             make_flags(name, value=getattr(arguments, name))
             for name in sorted(vars(arguments))
             if name not in excludes and not name.startswith('_')
-        )
+        ),
     )
 
 
@@ -50,7 +50,7 @@ def make_repository_flags(repository_path, local_borg_version):
     ) + (repository_path,)
 
 
-ARCHIVE_HASH_PATTERN = re.compile('[0-9a-fA-F]{8,}$')
+ARCHIVE_HASH_PATTERN = re.compile(r'[0-9a-fA-F]{8,}$')
 
 
 def make_repository_archive_flags(repository_path, archive, local_borg_version):
@@ -76,8 +76,8 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
     )
 
 
-DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
+DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'  # noqa: FS003
+DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'
 
 
 def get_default_archive_name_format(local_borg_version):
@@ -90,7 +90,7 @@ def get_default_archive_name_format(local_borg_version):
     return DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES
 
 
-def make_match_archives_flags(
+def make_match_archives_flags(  # noqa: PLR0911
     match_archives,
     archive_name_format,
     local_borg_version,
@@ -115,8 +115,8 @@ def make_match_archives_flags(
                 return ('--match-archives', f'aid:{match_archives}')
 
             return ('--match-archives', match_archives)
-        else:
+
-            return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
+        return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
 
     derived_match_archives = re.sub(
         r'\{(now|utcnow|pid)([:%\w\.-]*)\}',
@@ -131,8 +131,8 @@ def make_match_archives_flags(
 
     if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
         return ('--match-archives', f'sh:{derived_match_archives}')
-    else:
+
-        return ('--glob-archives', f'{derived_match_archives}')
+    return ('--glob-archives', f'{derived_match_archives}')
 
 
 def warn_for_aggressive_archive_flags(json_command, json_output):
@@ -150,7 +150,7 @@ def warn_for_aggressive_archive_flags(json_command, json_output):
         if len(json.loads(json_output)['archives']) == 0:
             logger.warning('An archive filter was applied, but no matching archives were found.')
             logger.warning(
-                'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.'
+                'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.',
             )
     except json.JSONDecodeError as error:
         logger.debug(f'Cannot parse JSON output from archive command: {error}')
@@ -193,8 +193,8 @@ def omit_flag_and_value(arguments, flag):
     # its value.
     return tuple(
         argument
-        for (previous_argument, argument) in zip((None,) + arguments, arguments)
+        for (previous_argument, argument) in zip((None, *arguments), arguments)
-        if flag not in (previous_argument, argument)
+        if flag not in {previous_argument, argument}
         if not argument.startswith(f'{flag}=')
     )
 
@@ -209,7 +209,7 @@ def make_exclude_flags(config):
         itertools.chain.from_iterable(
             ('--exclude-if-present', if_present)
             for if_present in config.get('exclude_if_present', ())
-        )
+        ),
     )
     keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
     exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
@@ -229,10 +229,10 @@ def make_list_filter_flags(local_borg_version, dry_run):
     if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
         if show_excludes or dry_run:
             return f'{base_flags}+-'
-        else:
+
-            return base_flags
+        return base_flags
 
     if show_excludes:
         return f'{base_flags}x-'
-    else:
+
-        return f'{base_flags}-'
+    return f'{base_flags}-'

borgmatic/borg/info.py

@@ -55,7 +55,8 @@ def make_info_command(
             )
         )
         + flags.make_flags_from_arguments(
-            info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
+            info_arguments,
+            excludes=('repository', 'archive', 'prefix', 'match_archives'),
         )
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
@@ -119,3 +120,5 @@ def display_archives_info(
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
+
+    return None

borgmatic/borg/list.py

@@ -17,7 +17,8 @@ MAKE_FLAGS_EXCLUDES = (
     'archive',
     'paths',
     'find_paths',
-) + ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST
+    *ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST,
+)
 
 
 def make_list_command(
@@ -53,7 +54,9 @@ def make_list_command(
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (
             flags.make_repository_archive_flags(
-                repository_path, list_arguments.archive, local_borg_version
+                repository_path,
+                list_arguments.archive,
+                local_borg_version,
             )
             if list_arguments.archive
             else flags.make_repository_flags(repository_path, local_borg_version)
@@ -115,10 +118,10 @@ def capture_archive_listing(
                 argparse.Namespace(
                     repository=repository_path,
                     archive=archive,
-                    paths=[path for path in list_paths] if list_paths else None,
+                    paths=list(list_paths) if list_paths else None,
                     find_paths=None,
                     json=None,
-                    format=path_format or '{path}{NUL}',  # noqa: FS003
+                    format=path_format or '{path}{NUL}',
                 ),
                 global_arguments,
                 local_path,
@@ -130,7 +133,7 @@ def capture_archive_listing(
             borg_exit_codes=config.get('borg_exit_codes'),
         )
         .strip('\0')
-        .split('\0')
+        .split('\0'),
     )
 
 
@@ -156,7 +159,7 @@ def list_archive(
     if not list_arguments.archive and not list_arguments.find_paths:
         if feature.available(feature.Feature.REPO_LIST, local_borg_version):
             logger.warning(
-                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.'
+                'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.',
             )
 
         repo_list_arguments = argparse.Namespace(
@@ -184,12 +187,12 @@ def list_archive(
         for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
             if getattr(list_arguments, name, None):
                 logger.warning(
-                    f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag."
+                    f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag.",
                 )
 
     if list_arguments.json:
         raise ValueError(
-            'The --json flag on the list action is not supported when using the --archive/--find flags.'
+            'The --json flag on the list action is not supported when using the --archive/--find flags.',
         )
 
     borg_exit_codes = config.get('borg_exit_codes')
@@ -227,7 +230,7 @@ def list_archive(
                 borg_exit_codes=borg_exit_codes,
             )
             .strip('\n')
-            .splitlines()
+            .splitlines(),
         )
     else:
         archive_lines = (list_arguments.archive,)
@@ -262,3 +265,5 @@ def list_archive(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
+
+    return None

borgmatic/borg/pattern.py

@@ -4,8 +4,6 @@ import logging
 import os
 import tempfile
 
-import borgmatic.borg.pattern
-
 logger = logging.getLogger(__name__)
 
 
@@ -59,9 +57,9 @@ Pattern = collections.namedtuple(
 
 def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
     '''
-    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
+    Given a sequence of patterns as Pattern instances, write them to a named temporary file in the
-    temporary file in the given borgmatic runtime directory and return the file object so it can
+    given borgmatic runtime directory and return the file object so it can continue to exist on disk
-    continue to exist on disk as long as the caller needs it.
+    as long as the caller needs it.
 
     If an optional open pattern file is given, append to it instead of making a new temporary file.
     Return None if no patterns are provided.
@@ -70,7 +68,9 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=Non
         return None
 
     if patterns_file is None:
-        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
+        patterns_file = tempfile.NamedTemporaryFile(
+            'w', dir=borgmatic_runtime_directory, encoding='utf-8'
+        )
         operation_name = 'Writing'
     else:
         patterns_file.write('\n')
@@ -90,17 +90,17 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=Non
 
 def check_all_root_patterns_exist(patterns):
     '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
+    Given a sequence of Pattern instances, check that all root pattern paths exist. If any don't,
-    paths exist. If any don't, raise an exception.
+    raise an exception.
     '''
     missing_paths = [
         pattern.path
         for pattern in patterns
-        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+        if pattern.type == Pattern_type.ROOT
         if not os.path.exists(pattern.path)
     ]
 
     if missing_paths:
         raise ValueError(
-            f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}"
+            f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}",
         )

borgmatic/borg/prune.py

@@ -1,4 +1,5 @@
 import logging
+import shlex
 
 import borgmatic.config.paths
 import borgmatic.logger
@@ -90,7 +91,7 @@ def prune_archives(
         + (('--list',) if config.get('list_details') else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 

borgmatic/borg/recreate.py

@@ -38,7 +38,8 @@ def recreate_archive(
 
     # Write patterns to a temporary file and use that file with --patterns-from.
     patterns_file = write_patterns_file(
-        patterns, borgmatic.config.paths.get_working_directory(config)
+        patterns,
+        borgmatic.config.paths.get_working_directory(config),
     )
 
     recreate_command = (
@@ -80,7 +81,8 @@ def recreate_archive(
                 )
             )
             if borgmatic.borg.feature.available(
-                borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version
+                borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE,
+                local_borg_version,
             )
             else (
                 flags.make_repository_archive_flags(repository, archive, local_borg_version)

borgmatic/borg/rename.py

@@ -0,0 +1,64 @@
+import logging
+
+import borgmatic.borg.flags
+
+logger = logging.getLogger(__name__)
+
+
+def make_rename_command(
+    dry_run,
+    repository_name,
+    old_archive_name,
+    new_archive_name,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    return (
+        (local_path, 'rename')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + borgmatic.borg.flags.make_flags('dry-run', dry_run)
+        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
+        + borgmatic.borg.flags.make_flags('umask', config.get('umask'))
+        + borgmatic.borg.flags.make_flags('log-json', config.get('log_json'))
+        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+        + borgmatic.borg.flags.make_repository_archive_flags(
+            repository_name,
+            old_archive_name,
+            local_borg_version,
+        )
+        + (new_archive_name,)
+    )
+
+
+def rename_archive(
+    repository_name,
+    old_archive_name,
+    new_archive_name,
+    dry_run,
+    config,
+    local_borg_version,
+    local_path,
+    remote_path,
+):
+    command = make_rename_command(
+        dry_run,
+        repository_name,
+        old_archive_name,
+        new_archive_name,
+        config,
+        local_borg_version,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.execute.execute_command(
+        command,
+        output_log_level=logging.INFO,
+        environment=borgmatic.borg.environment.make_environment(config),
+        working_directory=borgmatic.config.paths.get_working_directory(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/repo_create.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import logging
+import shlex
 import subprocess
 
 import borgmatic.config.paths
@@ -48,13 +49,13 @@ def create_repository(
                 global_arguments,
                 local_path,
                 remote_path,
-            )
+            ),
         )
         repository_encryption_mode = info_data.get('encryption', {}).get('mode')
 
         if repository_encryption_mode != encryption_mode:
             raise ValueError(
-                f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"'
+                f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"',
             )
 
         logger.info('Repository already exists. Skipping creation.')
@@ -86,12 +87,12 @@ def create_repository(
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     if dry_run:
-        logging.info('Skipping repository creation (dry run)')
+        logger.info('Skipping repository creation (dry run)')
         return
 
     # Do not capture output here, so as to support interactive prompts.

borgmatic/borg/repo_delete.py

@@ -9,6 +9,9 @@ import borgmatic.execute
 logger = logging.getLogger(__name__)
 
 
+FORCE_HARDER_FLAG_COUNT = 2
+
+
 def make_repo_delete_command(
     repository,
     config,
@@ -28,7 +31,8 @@ def make_repo_delete_command(
         + (
             ('repo-delete',)
             if borgmatic.borg.feature.available(
-                borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
+                borgmatic.borg.feature.Feature.REPO_DELETE,
+                local_borg_version,
             )
             else ('delete',)
         )
@@ -41,12 +45,16 @@ def make_repo_delete_command(
         + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
         + borgmatic.borg.flags.make_flags('list', config.get('list_details'))
         + (
-            (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ()))
+            (
+                ('--force',)
+                + (('--force',) if repo_delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
+            )
             if repo_delete_arguments.force
             else ()
         )
         + borgmatic.borg.flags.make_flags_from_arguments(
-            repo_delete_arguments, excludes=('list_details', 'force', 'repository')
+            repo_delete_arguments,
+            excludes=('list_details', 'force', 'repository'),
         )
         + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
     )

borgmatic/borg/repo_info.py

@@ -61,12 +61,14 @@ def display_repository_info(
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
-    else:
+
-        execute_command(
+    execute_command(
-            full_command,
+        full_command,
-            output_log_level=logging.ANSWER,
+        output_log_level=logging.ANSWER,
-            environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
-            working_directory=working_directory,
+        working_directory=working_directory,
-            borg_local_path=local_path,
+        borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+        borg_exit_codes=borg_exit_codes,
-        )
+    )
+
+    return None

borgmatic/borg/repo_list.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 import logging
 
 import borgmatic.config.paths
@@ -22,44 +23,83 @@ def resolve_archive_name(
     Given a local or remote repository path, an archive name, a configuration dict, the local Borg
     version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
     return the archive name. But if the archive name is "latest", then instead introspect the
-    repository for the latest archive and return its name.
+    repository for the latest archive and return its name or ID, depending on whether the version of
+    Borg in use supports archive series—different archives that share the same name but have unique
+    IDs.
 
     Raise ValueError if "latest" is given but there are no archives in the repository.
     '''
     if archive != 'latest':
         return archive
 
-    full_command = (
+    latest_archive = get_latest_archive(
-        (
+        repository_path,
-            local_path,
+        config,
-            (
+        local_borg_version,
-                'repo-list'
+        global_arguments,
-                if feature.available(feature.Feature.REPO_LIST, local_borg_version)
+        local_path=local_path,
-                else 'list'
+        remote_path=remote_path,
-            ),
-        )
-        + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('umask', config.get('umask'))
-        + flags.make_flags('log-json', config.get('log_json'))
-        + flags.make_flags('lock-wait', config.get('lock_wait'))
-        + flags.make_flags('last', 1)
-        + ('--short',)
-        + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    output = execute_command_and_capture_output(
+    return (
+        latest_archive['id']
+        if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
+        else latest_archive['name']
+    )
+
+
+def get_latest_archive(
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+    consider_checkpoints=False,
+):
+    '''
+    Returns a dict with information about the latest archive of a repository.
+
+    Raises ValueError if there are no archives in the repository.
+    '''
+
+    full_command = (
+        local_path,
+        (
+            'repo-list'
+            if feature.available(feature.Feature.REPO_LIST, local_borg_version)
+            else 'list'
+        ),
+        *flags.make_flags('remote-path', remote_path),
+        *flags.make_flags('umask', config.get('umask')),
+        *flags.make_flags('log-json', config.get('log_json')),
+        *flags.make_flags('lock-wait', config.get('lock_wait')),
+        *(
+            flags.make_flags('consider-checkpoints', consider_checkpoints)
+            if not feature.available(feature.Feature.REPO_LIST, local_borg_version)
+            else ()
+        ),
+        *flags.make_flags('last', 1),
+        '--json',
+        *flags.make_repository_flags(repository_path, local_borg_version),
+    )
+
+    json_output = execute_command_and_capture_output(
         full_command,
         environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
+
+    archives = json.loads(json_output)['archives']
+
     try:
-        latest_archive = output.strip().splitlines()[-1]
+        latest_archive = archives[-1]
     except IndexError:
         raise ValueError('No archives found in the repository')
 
-    logger.debug(f'Latest archive is {latest_archive}')
+    logger.debug(f'Latest archive is {latest_archive["name"]} ({latest_archive["id"]})')
 
     return latest_archive
 
@@ -183,3 +223,5 @@ def list_repository(
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
     )
+
+    return None

borgmatic/commands/arguments.py

@@ -60,7 +60,7 @@ def get_subactions_for_actions(action_parsers):
         action: tuple(
             subaction_name
             for group_action in action_parser._subparsers._group_actions
-            for subaction_name in group_action.choices.keys()
+            for subaction_name in group_action.choices
         )
         for action, action_parser in action_parsers.items()
         if action_parser._subparsers
@@ -77,21 +77,25 @@ def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments
     '''
     remaining_arguments = list(unparsed_arguments)
 
-    for action_name, parsed in parsed_arguments.items():
+    for parsed in parsed_arguments.values():
         for value in vars(parsed).values():
             if isinstance(value, str):
-                if value in ACTION_ALIASES.keys() and value in remaining_arguments:
+                if value in ACTION_ALIASES and value in remaining_arguments:
                     remaining_arguments.remove(value)
             elif isinstance(value, list):
                 for item in value:
-                    if item in ACTION_ALIASES.keys() and item in remaining_arguments:
+                    if item in ACTION_ALIASES and item in remaining_arguments:
                         remaining_arguments.remove(item)
 
     return tuple(remaining_arguments)
 
 
 def parse_and_record_action_arguments(
-    unparsed_arguments, parsed_arguments, action_parser, action_name, canonical_name=None
+    unparsed_arguments,
+    parsed_arguments,
+    action_parser,
+    action_name,
+    canonical_name=None,
 ):
     '''
     Given unparsed arguments as a sequence of strings, parsed arguments as a dict from action name
@@ -102,7 +106,8 @@ def parse_and_record_action_arguments(
     given action doesn't apply to the given unparsed arguments.
     '''
     filtered_arguments = omit_values_colliding_with_action_names(
-        unparsed_arguments, parsed_arguments
+        unparsed_arguments,
+        parsed_arguments,
     )
 
     if action_name not in filtered_arguments:
@@ -186,12 +191,12 @@ def get_unparsable_arguments(remaining_action_arguments):
         itertools.chain.from_iterable(
             argument_group
             for argument_group in dict.fromkeys(
-                itertools.chain.from_iterable(grouped_action_arguments)
+                itertools.chain.from_iterable(grouped_action_arguments),
-            ).keys()
+            )
             if all(
                 argument_group in action_arguments for action_arguments in grouped_action_arguments
             )
-        )
+        ),
     )
 
 
@@ -244,7 +249,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
                             subaction_name,
                         )
                         if argument != action_name
-                    )
+                    ),
                 )
 
                 if subaction_name in arguments:
@@ -256,14 +261,18 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
                     sys.exit(0)
                 else:
                     raise ValueError(
-                        f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}"
+                        f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}",
                     )
         # Otherwise, parse with the main action parser.
         else:
             remaining_action_arguments.append(
                 parse_and_record_action_arguments(
-                    unparsed_arguments, arguments, action_parser, action_name, canonical_name
+                    unparsed_arguments,
-                )
+                    arguments,
+                    action_parser,
+                    action_name,
+                    canonical_name,
+                ),
             )
 
     # If no actions were explicitly requested, assume defaults.
@@ -272,11 +281,11 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
             default_action_parser = action_parsers[default_action_name]
             remaining_action_arguments.append(
                 parse_and_record_action_arguments(
-                    tuple(unparsed_arguments) + (default_action_name,),
+                    (*unparsed_arguments, default_action_name),
                     arguments,
                     default_action_parser,
                     default_action_name,
-                )
+                ),
             )
 
     arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments)
@@ -304,10 +313,10 @@ def make_argument_description(schema, flag_name):
 
     if '[0]' in flag_name:
         pieces.append(
-            ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).'
+            ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).',
         )
 
-    if example and schema_type in ('array', 'object'):
+    if example and schema_type in ('array', 'object'):  # noqa: PLR6201
         example_buffer = io.StringIO()
         yaml = ruamel.yaml.YAML(typ='safe')
         yaml.default_flow_style = True
@@ -387,7 +396,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name):
         if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name:
             continue
 
-        if action_registry_name in ('store_true', 'store_false'):
+        if action_registry_name in {'store_true', 'store_false'}:
             arguments_group.add_argument(
                 unparsed_flag_name,
                 action=action_registry_name,
@@ -408,7 +417,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name):
             )
 
 
-def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None):
+def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None):  # noqa: PLR0912
     '''
     Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of
     unparsed argument strings, convert the entire schema into corresponding command-line flags and
@@ -466,7 +475,10 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
         if properties:
             for name, child in properties.items():
                 add_arguments_from_schema(
-                    arguments_group, child, unparsed_arguments, names + (name,)
+                    arguments_group,
+                    child,
+                    unparsed_arguments,
+                    (*names, name),
                 )
 
             return
@@ -483,12 +495,15 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
                     arguments_group,
                     child,
                     unparsed_arguments,
-                    names[:-1] + (f'{names[-1]}[0]',) + (name,),
+                    (*names[:-1], f'{names[-1]}[0]', name),
                 )
         # If there aren't any children, then this is an array of scalars. Recurse accordingly.
         else:
             add_arguments_from_schema(
-                arguments_group, items, unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',)
+                arguments_group,
+                items,
+                unparsed_arguments,
+                (*names[:-1], f'{names[-1]}[0]'),
             )
 
     flag_name = '.'.join(names).replace('_', '-')
@@ -515,9 +530,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
         )
 
         if names[-1].startswith('no_'):
-            no_flag_name = '.'.join(names[:-1] + (names[-1][len('no_') :],)).replace('_', '-')
+            no_flag_name = '.'.join((*names[:-1], names[-1][len('no_') :])).replace('_', '-')
         else:
-            no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-')
+            no_flag_name = '.'.join((*names[:-1], 'no-' + names[-1])).replace('_', '-')
 
         arguments_group.add_argument(
             f'--{no_flag_name}',
@@ -545,7 +560,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
     add_array_element_arguments(arguments_group, unparsed_arguments, flag_name)
 
 
-def make_parsers(schema, unparsed_arguments):
+def make_parsers(schema, unparsed_arguments):  # noqa: PLR0915
     '''
     Given a configuration schema dict and unparsed arguments as a sequence of strings, build a
     global arguments parser, individual action parsers, and a combined parser containing both.
@@ -667,10 +682,13 @@ def make_parsers(schema, unparsed_arguments):
         dest='make_parent_directories',
         default=None,
         action='store_true',
-        help='Create any missing parent directories of the repository directory',
+        help='Create any missing parent directories of the repository directory [Borg 1.x only]',
     )
     repo_create_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     transfer_parser = action_parsers.add_parser(
@@ -712,7 +730,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Only transfer archives with names, hashes, or series matching this pattern',
     )
     transfer_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     transfer_group.add_argument(
         '--first',
@@ -720,7 +740,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Only transfer first N archives after other filters are applied',
     )
     transfer_group.add_argument(
-        '--last', metavar='N', help='Only transfer last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Only transfer last N archives after other filters are applied',
     )
     transfer_group.add_argument(
         '--oldest',
@@ -743,7 +765,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Transfer archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
     )
     transfer_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     prune_parser = action_parsers.add_parser(
@@ -833,7 +858,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Minimum saved space percentage threshold for compacting a segment, defaults to 10',
     )
     compact_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     create_parser = action_parsers.add_parser(
@@ -870,7 +898,16 @@ def make_parsers(schema, unparsed_arguments):
         help='Show per-file details',
     )
     create_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        dest='json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
+    )
+    create_group.add_argument(
+        '--comment',
+        metavar='COMMENT',
+        help='Add a comment text to the archive',
     )
     create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
@@ -991,13 +1028,19 @@ def make_parsers(schema, unparsed_arguments):
         help='Only delete archives with names, hashes, or series matching this pattern',
     )
     delete_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     delete_group.add_argument(
-        '--first', metavar='N', help='Delete first N archives after other filters are applied'
+        '--first',
+        metavar='N',
+        help='Delete first N archives after other filters are applied',
     )
     delete_group.add_argument(
-        '--last', metavar='N', help='Delete last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Delete last N archives after other filters are applied',
     )
     delete_group.add_argument(
         '--oldest',
@@ -1034,7 +1077,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to extract, defaults to the configured repository if there is only one, quoted globs supported',
     )
     extract_group.add_argument(
-        '--archive', help='Name or hash of a single archive to extract (or "latest")', required=True
+        '--archive',
+        help='Name or hash of a single archive to extract (or "latest")',
+        required=True,
     )
     extract_group.add_argument(
         '--path',
@@ -1063,7 +1108,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Display progress for each file as it is extracted',
     )
     extract_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     config_parser = action_parsers.add_parser(
@@ -1088,7 +1136,7 @@ def make_parsers(schema, unparsed_arguments):
         add_help=False,
     )
     config_bootstrap_group = config_bootstrap_parser.add_argument_group(
-        'config bootstrap arguments'
+        'config bootstrap arguments',
     )
     config_bootstrap_group.add_argument(
         '--repository',
@@ -1143,7 +1191,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Command to use instead of "ssh"',
     )
     config_bootstrap_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     config_generate_parser = config_parsers.add_parser(
@@ -1173,7 +1224,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Whether to overwrite any existing destination file, defaults to false',
     )
     config_generate_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     config_validate_parser = config_parsers.add_parser(
@@ -1190,7 +1244,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Show the validated configuration after all include merging has occurred',
     )
     config_validate_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     export_tar_parser = action_parsers.add_parser(
@@ -1206,7 +1263,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to export from, defaults to the configured repository if there is only one, quoted globs supported',
     )
     export_tar_group.add_argument(
-        '--archive', help='Name or hash of a single archive to export (or "latest")', required=True
+        '--archive',
+        help='Name or hash of a single archive to export (or "latest")',
+        required=True,
     )
     export_tar_group.add_argument(
         '--path',
@@ -1223,7 +1282,8 @@ def make_parsers(schema, unparsed_arguments):
         required=True,
     )
     export_tar_group.add_argument(
-        '--tar-filter', help='Name of filter program to pipe data through'
+        '--tar-filter',
+        help='Name of filter program to pipe data through',
     )
     export_tar_group.add_argument(
         '--list',
@@ -1241,7 +1301,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Number of leading path components to remove from each exported path. Skip paths with fewer elements',
     )
     export_tar_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     mount_parser = action_parsers.add_parser(
@@ -1257,7 +1320,8 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to use, defaults to the configured repository if there is only one, quoted globs supported',
     )
     mount_group.add_argument(
-        '--archive', help='Name or hash of a single archive to mount (or "latest")'
+        '--archive',
+        help='Name or hash of a single archive to mount (or "latest")',
     )
     mount_group.add_argument(
         '--mount-point',
@@ -1286,7 +1350,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Mount first N archives after other filters are applied',
     )
     mount_group.add_argument(
-        '--last', metavar='N', help='Mount last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Mount last N archives after other filters are applied',
     )
     mount_group.add_argument(
         '--oldest',
@@ -1363,7 +1429,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Do not delete the local security info when deleting a repository',
     )
     repo_delete_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     restore_parser = action_parsers.add_parser(
@@ -1406,6 +1475,10 @@ def make_parsers(schema, unparsed_arguments):
         '--port',
         help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--container',
+        help='Container to restore to. Defaults to the "restore_container" option in borgmatic\'s configuration',
+    )
     restore_group.add_argument(
         '--username',
         help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
@@ -1418,10 +1491,18 @@ def make_parsers(schema, unparsed_arguments):
         '--restore-path',
         help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--original-label',
+        help='The label where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '--original-hostname',
         help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
     )
+    restore_group.add_argument(
+        '--original-container',
+        help='The container where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '--original-port',
         type=int,
@@ -1432,7 +1513,10 @@ def make_parsers(schema, unparsed_arguments):
         help='The name of the data source hook for the dump to restore, only necessary if you need to disambiguate dumps',
     )
     restore_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     repo_list_parser = action_parsers.add_parser(
@@ -1448,14 +1532,22 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to list, defaults to the configured repositories, quoted globs supported',
     )
     repo_list_group.add_argument(
-        '--short', default=False, action='store_true', help='Output only archive names'
+        '--short',
+        default=False,
+        action='store_true',
+        help='Output only archive names',
     )
     repo_list_group.add_argument('--format', help='Format for archive listing')
     repo_list_group.add_argument(
-        '--json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     repo_list_group.add_argument(
-        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
+        '-P',
+        '--prefix',
+        help='Deprecated. Only list archive names starting with this prefix',
     )
     repo_list_group.add_argument(
         '-a',
@@ -1465,13 +1557,19 @@ def make_parsers(schema, unparsed_arguments):
         help='Only list archive names, hashes, or series matching this pattern',
     )
     repo_list_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     repo_list_group.add_argument(
-        '--first', metavar='N', help='List first N archives after other filters are applied'
+        '--first',
+        metavar='N',
+        help='List first N archives after other filters are applied',
     )
     repo_list_group.add_argument(
-        '--last', metavar='N', help='List last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='List last N archives after other filters are applied',
     )
     repo_list_group.add_argument(
         '--oldest',
@@ -1500,7 +1598,10 @@ def make_parsers(schema, unparsed_arguments):
         help="List only deleted archives that haven't yet been compacted [Borg 2.x+ only]",
     )
     repo_list_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     list_parser = action_parsers.add_parser(
@@ -1516,7 +1617,8 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository containing archive to list, defaults to the configured repositories, quoted globs supported',
     )
     list_group.add_argument(
-        '--archive', help='Name or hash of a single archive to list (or "latest")'
+        '--archive',
+        help='Name or hash of a single archive to list (or "latest")',
     )
     list_group.add_argument(
         '--path',
@@ -1533,14 +1635,22 @@ def make_parsers(schema, unparsed_arguments):
         help='Partial path or pattern to search for and list across multiple archives, can specify flag multiple times',
     )
     list_group.add_argument(
-        '--short', default=False, action='store_true', help='Output only path names'
+        '--short',
+        default=False,
+        action='store_true',
+        help='Output only path names',
     )
     list_group.add_argument('--format', help='Format for file listing')
     list_group.add_argument(
-        '--json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     list_group.add_argument(
-        '-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
+        '-P',
+        '--prefix',
+        help='Deprecated. Only list archive names starting with this prefix',
     )
     list_group.add_argument(
         '-a',
@@ -1550,19 +1660,30 @@ def make_parsers(schema, unparsed_arguments):
         help='Only list archive names matching this pattern',
     )
     list_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     list_group.add_argument(
-        '--first', metavar='N', help='List first N archives after other filters are applied'
+        '--first',
+        metavar='N',
+        help='List first N archives after other filters are applied',
     )
     list_group.add_argument(
-        '--last', metavar='N', help='List last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='List last N archives after other filters are applied',
     )
     list_group.add_argument(
-        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+        '-e',
+        '--exclude',
+        metavar='PATTERN',
+        help='Exclude paths matching the pattern',
     )
     list_group.add_argument(
-        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+        '--exclude-from',
+        metavar='FILENAME',
+        help='Exclude paths from exclude file, one per line',
     )
     list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
     list_group.add_argument(
@@ -1585,10 +1706,17 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to show info for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     repo_info_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        dest='json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     repo_info_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     info_parser = action_parsers.add_parser(
@@ -1604,10 +1732,15 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     info_group.add_argument(
-        '--archive', help='Archive name, hash, or series to show info for (or "latest")'
+        '--archive',
+        help='Archive name, hash, or series to show info for (or "latest")',
     )
     info_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--json',
+        dest='json',
+        default=False,
+        action='store_true',
+        help='Output results as JSON',
     )
     info_group.add_argument(
         '-P',
@@ -1622,7 +1755,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Only show info for archive names, hashes, or series matching this pattern',
     )
     info_group.add_argument(
-        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+        '--sort-by',
+        metavar='KEYS',
+        help='Comma-separated list of sorting keys',
     )
     info_group.add_argument(
         '--first',
@@ -1630,7 +1765,9 @@ def make_parsers(schema, unparsed_arguments):
         help='Show info for first N archives after other filters are applied',
     )
     info_group.add_argument(
-        '--last', metavar='N', help='Show info for last N archives after other filters are applied'
+        '--last',
+        metavar='N',
+        help='Show info for last N archives after other filters are applied',
     )
     info_group.add_argument(
         '--oldest',
@@ -1667,7 +1804,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to break the lock for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     break_lock_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     key_parser = action_parsers.add_parser(
@@ -1712,7 +1852,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Path to export the key to, defaults to stdout (but be careful about dirtying the output with --verbosity)',
     )
     key_export_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     key_import_parser = key_parsers.add_parser(
@@ -1737,7 +1880,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Path to import the key from backup, defaults to stdin',
     )
     key_import_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     key_change_passphrase_parser = key_parsers.add_parser(
@@ -1747,14 +1893,17 @@ def make_parsers(schema, unparsed_arguments):
         add_help=False,
     )
     key_change_passphrase_group = key_change_passphrase_parser.add_argument_group(
-        'key change-passphrase arguments'
+        'key change-passphrase arguments',
     )
     key_change_passphrase_group.add_argument(
         '--repository',
         help='Path of repository to change the passphrase for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     key_change_passphrase_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     recreate_parser = action_parsers.add_parser(
@@ -1804,7 +1953,10 @@ def make_parsers(schema, unparsed_arguments):
         help='Only consider archive names, hashes, or series matching this pattern [Borg 2.x+ only]',
     )
     recreate_group.add_argument(
-        '-h', '--help', action='help', help='Show this help message and exit'
+        '-h',
+        '--help',
+        action='help',
+        help='Show this help message and exit',
     )
 
     borg_parser = action_parsers.add_parser(
@@ -1820,7 +1972,8 @@ def make_parsers(schema, unparsed_arguments):
         help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported',
     )
     borg_group.add_argument(
-        '--archive', help='Archive name, hash, or series to pass to Borg (or "latest")'
+        '--archive',
+        help='Archive name, hash, or series to pass to Borg (or "latest")',
     )
     borg_group.add_argument(
         '--',
@@ -1834,6 +1987,9 @@ def make_parsers(schema, unparsed_arguments):
     return global_parser, action_parsers, global_plus_action_parser
 
 
+HIGHLANDER_ACTION_ARGUMENTS_COUNT = 2  # 1 for "global" + 1 for the action
+
+
 def parse_arguments(schema, *unparsed_arguments):
     '''
     Given a configuration schema dict and the command-line arguments with which this script was
@@ -1844,21 +2000,22 @@ def parse_arguments(schema, *unparsed_arguments):
     Raise SystemExit with an error code of 0 if "--help" was requested.
     '''
     global_parser, action_parsers, global_plus_action_parser = make_parsers(
-        schema, unparsed_arguments
+        schema,
+        unparsed_arguments,
     )
     arguments, remaining_action_arguments = parse_arguments_for_actions(
-        unparsed_arguments, action_parsers.choices, global_parser
+        unparsed_arguments,
+        action_parsers.choices,
+        global_parser,
     )
 
     if not arguments['global'].config_paths:
         arguments['global'].config_paths = collect.get_default_config_paths(expand_home=True)
 
     for action_name in ('bootstrap', 'generate', 'validate'):
-        if (
+        if action_name in arguments and len(arguments) > HIGHLANDER_ACTION_ARGUMENTS_COUNT:
-            action_name in arguments.keys() and len(arguments.keys()) > 2
-        ):  # 2 = 1 for 'global' + 1 for the action
             raise ValueError(
-                f'The {action_name} action cannot be combined with other actions. Please run it separately.'
+                f'The {action_name} action cannot be combined with other actions. Please run it separately.',
             )
 
     unknown_arguments = get_unparsable_arguments(remaining_action_arguments)
@@ -1870,11 +2027,11 @@ def parse_arguments(schema, *unparsed_arguments):
 
         global_plus_action_parser.print_usage()
         raise ValueError(
-            f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}"
+            f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}",
         )
 
     if (
-        ('list' in arguments and 'repo-info' in arguments and arguments['list'].json)
+        ('list' in arguments and 'repo-info' in arguments and arguments['list'].json)  # noqa: PLR0916
         or ('list' in arguments and 'info' in arguments and arguments['list'].json)
         or ('repo-info' in arguments and 'info' in arguments and arguments['repo-info'].json)
     ):
@@ -1882,23 +2039,23 @@ def parse_arguments(schema, *unparsed_arguments):
 
     if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives):
         raise ValueError(
-            'With the list action, only one of --prefix or --match-archives flags can be used.'
+            'With the list action, only one of --prefix or --match-archives flags can be used.',
         )
 
     if 'repo-list' in arguments and (
         arguments['repo-list'].prefix and arguments['repo-list'].match_archives
     ):
         raise ValueError(
-            'With the repo-list action, only one of --prefix or --match-archives flags can be used.'
+            'With the repo-list action, only one of --prefix or --match-archives flags can be used.',
         )
 
-    if 'info' in arguments and (
+    if 'info' in arguments and (  # noqa: PLR0916
         (arguments['info'].archive and arguments['info'].prefix)
         or (arguments['info'].archive and arguments['info'].match_archives)
         or (arguments['info'].prefix and arguments['info'].match_archives)
     ):
         raise ValueError(
-            'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.'
+            'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.',
         )
 
     if 'borg' in arguments and arguments['global'].dry_run:

borgmatic/commands/borgmatic.py

@@ -95,7 +95,9 @@ class Monitoring_hooks:
         self.config_filename = config_filename
         self.config = config
         self.dry_run = global_arguments.dry_run
-        self.monitoring_log_level = verbosity_to_log_level(config.get('monitoring_verbosity'))
+        self.monitoring_log_level = verbosity_to_log_level(
+            get_verbosity({config_filename: config}, 'monitoring_verbosity'),
+        )
         self.monitoring_hooks_are_activated = (
             using_primary_action and self.monitoring_log_level != DISABLED
         )
@@ -180,7 +182,7 @@ class Monitoring_hooks:
         )
 
 
-def run_configuration(config_filename, config, config_paths, arguments):
+def run_configuration(config_filename, config, config_paths, arguments):  # noqa: PLR0912, PLR0915
     '''
     Given a config filename, the corresponding parsed config dict, a sequence of loaded
     configuration paths, and command-line arguments as a dict from subparser name to a namespace of
@@ -204,12 +206,13 @@ def run_configuration(config_filename, config, config_paths, arguments):
 
     if skip_actions:
         logger.debug(
-            f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions"
+            f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions",
         )
 
-    try:
+    try:  # noqa: PLR1702
-        with Monitoring_hooks(config_filename, config, arguments, global_arguments):
+        with (
-            with borgmatic.hooks.command.Before_after_hooks(
+            Monitoring_hooks(config_filename, config, arguments, global_arguments),
+            borgmatic.hooks.command.Before_after_hooks(
                 command_hooks=config.get('commands'),
                 before_after='configuration',
                 umask=config.get('umask'),
@@ -218,69 +221,80 @@ def run_configuration(config_filename, config, config_paths, arguments):
                 action_names=arguments.keys(),
                 configuration_filename=config_filename,
                 log_file=config.get('log_file', ''),
-            ):
+            ),
-                try:
+        ):
-                    local_borg_version = borg_version.local_borg_version(config, local_path)
+            try:
-                    logger.debug(f'Borg {local_borg_version}')
+                local_borg_version = borg_version.local_borg_version(config, local_path)
-                except (OSError, CalledProcessError, ValueError) as error:
+                logger.debug(f'Borg {local_borg_version}')
-                    yield from log_error_records(
+            except (OSError, CalledProcessError, ValueError) as error:
-                        f'{config_filename}: Error getting local Borg version', error
+                yield from log_error_records(
-                    )
+                    f'{config_filename}: Error getting local Borg version',
-                    return
+                    error,
+                )
+                raise
 
-                for repo in config['repositories']:
+            for repo in config['repositories']:
-                    repo_queue.put(
+                repo_queue.put(
-                        (repo, 0),
+                    (repo, 0),
-                    )
+                )
 
-                while not repo_queue.empty():
+            while not repo_queue.empty():
-                    repository, retry_num = repo_queue.get()
+                repository, retry_num = repo_queue.get()
 
-                    with Log_prefix(repository.get('label', repository['path'])):
+                with Log_prefix(repository.get('label', repository['path'])):
-                        logger.debug('Running actions for repository')
+                    logger.debug('Running actions for repository')
-                        timeout = retry_num * retry_wait
+                    timeout = retry_num * retry_wait
-                        if timeout:
+
-                            logger.warning(f'Sleeping {timeout}s before next retry')
+                    if timeout:
-                            time.sleep(timeout)
+                        logger.warning(f'Sleeping {timeout}s before next retry')
-                        try:
+                        time.sleep(timeout)
-                            yield from run_actions(
+
-                                arguments=arguments,
+                    try:
-                                config_filename=config_filename,
+                        yield from run_actions(
-                                config=config,
+                            arguments=arguments,
-                                config_paths=config_paths,
+                            config_filename=config_filename,
-                                local_path=local_path,
+                            config=config,
-                                remote_path=remote_path,
+                            config_paths=config_paths,
-                                local_borg_version=local_borg_version,
+                            local_path=local_path,
-                                repository=repository,
+                            remote_path=remote_path,
+                            local_borg_version=local_borg_version,
+                            repository=repository,
+                        )
+                    except (OSError, CalledProcessError, ValueError) as error:
+                        if retry_num < retries:
+                            repo_queue.put(
+                                (repository, retry_num + 1),
                             )
-                        except (OSError, CalledProcessError, ValueError) as error:
+                            tuple(  # Consume the generator so as to trigger logging.
-                            if retry_num < retries:
+                                log_error_records(
-                                repo_queue.put(
+                                    'Error running actions for repository',
-                                    (repository, retry_num + 1),
+                                    error,
-                                )
+                                    levelno=logging.WARNING,
-                                tuple(  # Consume the generator so as to trigger logging.
+                                    log_command_error_output=True,
-                                    log_error_records(
+                                ),
-                                        'Error running actions for repository',
-                                        error,
-                                        levelno=logging.WARNING,
-                                        log_command_error_output=True,
-                                    )
-                                )
-                                logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}')
-                                continue
-
-                            if command.considered_soft_failure(error):
-                                continue
-
-                            yield from log_error_records(
-                                'Error running actions for repository',
-                                error,
                             )
-                            encountered_error = error
+                            logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}')
-                            error_repository = repository
+                            continue
+
+                        if command.considered_soft_failure(error):
+                            continue
+
+                        yield from log_error_records(
+                            'Error running actions for repository',
+                            error,
+                        )
+                        encountered_error = error
+                        error_repository = repository
+
+            # Re-raise any error, so that the Monitoring_hooks context manager wrapping this
+            # code can see the error and act accordingly. Do this here rather than as soon as
+            # the error is encountered so that an error with one repository doesn't prevent
+            # other repositories from running.
+            if encountered_error:
+                raise encountered_error
 
     except (OSError, CalledProcessError, ValueError) as error:
-        yield from log_error_records('Error running configuration', error)
+        yield from log_error_records('Error running configuration')
 
         encountered_error = error
 
@@ -290,7 +304,10 @@ def run_configuration(config_filename, config, config_paths, arguments):
     try:
         command.execute_hooks(
             command.filter_hooks(
-                config.get('commands'), after='error', action_names=arguments.keys()
+                config.get('commands'),
+                after='error',
+                action_names=arguments.keys(),
+                state_names=['fail'],
             ),
             config.get('umask'),
             borgmatic.config.paths.get_working_directory(config),
@@ -309,7 +326,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
         yield from log_error_records(f'{config_filename}: Error running after error hook', error)
 
 
-def run_actions(
+def run_actions(  # noqa: PLR0912, PLR0915
     *,
     arguments,
     config_filename,
@@ -446,6 +463,7 @@ def run_actions(
                             global_arguments,
                             local_path,
                             remote_path,
+                            hook_context,
                         )
                 elif action_name == 'extract':
                     borgmatic.actions.extract.run_extract(
@@ -627,9 +645,9 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
                         levelno=logging.DEBUG,
                         levelname='DEBUG',
                         msg=f'{config_filename}: Loading configuration file',
-                    )
+                    ),
                 ),
-            ]
+            ],
         )
         try:
             configs[config_filename], paths, parse_logs = validate.parse_configuration(
@@ -646,12 +664,12 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
                 [
                     logging.makeLogRecord(
                         dict(
-                            levelno=logging.WARNING,
+                            levelno=logging.CRITICAL,
-                            levelname='WARNING',
+                            levelname='CRITICAL',
                             msg=f'{config_filename}: Insufficient permissions to read configuration file',
-                        )
+                        ),
                     ),
-                ]
+                ],
             )
         except (ValueError, OSError, validate.Validation_error) as error:
             logs.extend(
@@ -661,12 +679,12 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
                             levelno=logging.CRITICAL,
                             levelname='CRITICAL',
                             msg=f'{config_filename}: Error parsing configuration file',
-                        )
+                        ),
                     ),
                     logging.makeLogRecord(
-                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=str(error))
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=str(error)),
                     ),
-                ]
+                ],
             )
 
     return (configs, sorted(config_paths), logs)
@@ -689,7 +707,10 @@ BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE = 62
 
 
 def log_error_records(
-    message, error=None, levelno=logging.CRITICAL, log_command_error_output=False
+    message,
+    error=None,
+    levelno=logging.CRITICAL,
+    log_command_error_output=False,
 ):
     '''
     Given error message text, an optional exception object, an optional log level, and whether to
@@ -707,14 +728,14 @@ def log_error_records(
 
     try:
         raise error
-    except CalledProcessError as error:
+    except CalledProcessError as called_process_error:
         yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
 
-        if error.output:
+        if called_process_error.output:
             try:
-                output = error.output.decode('utf-8')
+                output = called_process_error.output.decode('utf-8')
             except (UnicodeDecodeError, AttributeError):
-                output = error.output
+                output = called_process_error.output
 
             # Suppress these logs for now and save the error output for the log summary at the end.
             # Log a separate record per line, as some errors can be really verbose and overflow the
@@ -727,18 +748,18 @@ def log_error_records(
                     suppress_log=True,
                 )
 
-        yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(called_process_error))
 
-        if error.returncode == BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE:
+        if called_process_error.returncode == BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE:
             yield log_record(
                 levelno=levelno,
                 levelname=level_name,
                 msg='\nTo work around this, set either the "relocated_repo_access_is_ok" or "unknown_unencrypted_repo_access_is_ok" option to "true", as appropriate.',
             )
-    except (ValueError, OSError) as error:
+    except (ValueError, OSError) as other_error:
         yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
-        yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(other_error))
-    except:  # noqa: E722
+    except:  # noqa: E722, S110
         # Raising above only as a means of determining the error type. Swallow the exception here
         # because we don't want the exception to propagate out of this function.
         pass
@@ -769,7 +790,8 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
         try:
             # No configuration file is needed for bootstrap.
             local_borg_version = borg_version.local_borg_version(
-                {}, arguments['bootstrap'].local_path
+                {},
+                arguments['bootstrap'].local_path,
             )
         except (OSError, CalledProcessError, ValueError) as error:
             yield from log_error_records('Error getting local Borg version', error)
@@ -777,14 +799,16 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
 
         try:
             borgmatic.actions.config.bootstrap.run_bootstrap(
-                arguments['bootstrap'], arguments['global'], local_borg_version
+                arguments['bootstrap'],
+                arguments['global'],
+                local_borg_version,
             )
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.ANSWER,
                     levelname='ANSWER',
                     msg='Bootstrap successful',
-                )
+                ),
             )
         except (
             CalledProcessError,
@@ -798,14 +822,15 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
     if 'generate' in arguments:
         try:
             borgmatic.actions.config.generate.run_generate(
-                arguments['generate'], arguments['global']
+                arguments['generate'],
+                arguments['global'],
             )
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.ANSWER,
                     levelname='ANSWER',
                     msg='Generate successful',
-                )
+                ),
             )
         except (
             CalledProcessError,
@@ -823,7 +848,7 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
                     levelno=logging.CRITICAL,
                     levelname='CRITICAL',
                     msg='Configuration validation failed',
-                )
+                ),
             )
 
             return
@@ -836,7 +861,7 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
                     levelno=logging.ANSWER,
                     levelname='ANSWER',
                     msg='All configuration files are valid',
-                )
+                ),
             )
         except (
             CalledProcessError,
@@ -848,7 +873,7 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
         return
 
 
-def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path):
+def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path):  # noqa: PLR0912
     '''
     Given a dict of configuration filename to corresponding parsed configuration, a sequence of
     loaded configuration paths, parsed command-line arguments as a dict from subparser name to a
@@ -861,9 +886,9 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
     # Run cross-file validation checks.
     repository = None
 
-    for action_name, action_arguments in arguments.items():
+    for action_arguments in arguments.values():
         if hasattr(action_arguments, 'repository'):
-            repository = getattr(action_arguments, 'repository')
+            repository = action_arguments.repository
             break
 
     try:
@@ -879,23 +904,36 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
         return
 
     try:
+        seen_command_hooks = []
+
         for config_filename, config in configs.items():
-            command.execute_hooks(
+            command_hooks = command.filter_hooks(
-                command.filter_hooks(
+                tuple(
-                    config.get('commands'), before='everything', action_names=arguments.keys()
+                    command_hook
+                    for command_hook in config.get('commands', ())
+                    if command_hook not in seen_command_hooks
                 ),
-                config.get('umask'),
+                before='everything',
-                borgmatic.config.paths.get_working_directory(config),
+                action_names=arguments.keys(),
-                arguments['global'].dry_run,
-                configuration_filename=config_filename,
-                log_file=log_file_path or '',
             )
+
+            if command_hooks:
+                command.execute_hooks(
+                    command_hooks,
+                    config.get('umask'),
+                    borgmatic.config.paths.get_working_directory(config),
+                    arguments['global'].dry_run,
+                    configuration_filename=config_filename,
+                    log_file=log_file_path or '',
+                )
+                seen_command_hooks += list(command_hooks)
     except (CalledProcessError, ValueError, OSError) as error:
         yield from log_error_records('Error running before everything hook', error)
         return
 
     # Execute the actions corresponding to each configuration file.
     json_results = []
+    encountered_error = False
 
     for config_filename, config in configs.items():
         with Log_prefix(config_filename):
@@ -906,6 +944,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
             )
 
             if error_logs:
+                encountered_error = True
                 yield from log_error_records('An error occurred')
                 yield from error_logs
             else:
@@ -914,7 +953,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
                         levelno=logging.INFO,
                         levelname='INFO',
                         msg=f'{config_filename}: Successfully ran configuration file',
-                    )
+                    ),
                 )
                 if results:
                     json_results.extend(results)
@@ -928,23 +967,37 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
                 local_path=get_local_path(configs),
             )
         except (CalledProcessError, OSError) as error:
+            encountered_error = True
             yield from log_error_records('Error unmounting mount point', error)
 
     if json_results:
         sys.stdout.write(json.dumps(json_results))
 
     try:
+        seen_command_hooks = []
+
         for config_filename, config in configs.items():
-            command.execute_hooks(
+            command_hooks = command.filter_hooks(
-                command.filter_hooks(
+                tuple(
-                    config.get('commands'), after='everything', action_names=arguments.keys()
+                    command_hook
+                    for command_hook in config.get('commands', ())
+                    if command_hook not in seen_command_hooks
                 ),
-                config.get('umask'),
+                after='everything',
-                borgmatic.config.paths.get_working_directory(config),
+                action_names=arguments.keys(),
-                arguments['global'].dry_run,
+                state_names=['fail' if encountered_error else 'finish'],
-                configuration_filename=config_filename,
-                log_file=log_file_path or '',
             )
+
+            if command_hooks:
+                command.execute_hooks(
+                    command_hooks,
+                    config.get('umask'),
+                    borgmatic.config.paths.get_working_directory(config),
+                    arguments['global'].dry_run,
+                    configuration_filename=config_filename,
+                    log_file=log_file_path or '',
+                )
+                seen_command_hooks += list(command_hooks)
     except (CalledProcessError, ValueError, OSError) as error:
         yield from log_error_records('Error running after everything hook', error)
 
@@ -989,17 +1042,17 @@ def get_singular_option_value(configs, option_name):
         configure_logging(logging.CRITICAL)
         joined_values = ', '.join(str(value) for value in distinct_values)
         logger.critical(
-            f'The {option_name} option has conflicting values across configuration files: {joined_values}'
+            f'The {option_name} option has conflicting values across configuration files: {joined_values}',
         )
         exit_with_help_link()
 
     try:
-        return tuple(distinct_values)[0]
+        return next(iter(distinct_values))
-    except IndexError:
+    except StopIteration:
         return None
 
 
-def main(extra_summary_logs=[]):  # pragma: no cover
+def main(extra_summary_logs=()):  # pragma: no cover
     configure_signals()
     configure_delayed_logging()
     schema_filename = validate.schema_filename()
@@ -1020,19 +1073,23 @@ def main(extra_summary_logs=[]):  # pragma: no cover
     except SystemExit as error:
         if error.code == 0:
             raise error
+
         configure_logging(logging.CRITICAL)
         logger.critical(f"Error parsing arguments: {' '.join(sys.argv)}")
         exit_with_help_link()
 
     global_arguments = arguments['global']
+
     if global_arguments.version:
-        print(importlib.metadata.version('borgmatic'))
+        print(importlib.metadata.version('borgmatic'))  # noqa: T201
         sys.exit(0)
+
     if global_arguments.bash_completion:
-        print(borgmatic.commands.completion.bash.bash_completion())
+        print(borgmatic.commands.completion.bash.bash_completion())  # noqa: T201
         sys.exit(0)
+
     if global_arguments.fish_completion:
-        print(borgmatic.commands.completion.fish.fish_completion())
+        print(borgmatic.commands.completion.fish.fish_completion())  # noqa: T201
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
@@ -1071,18 +1128,23 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         exit_with_help_link()
 
     summary_logs = (
-        extra_summary_logs
+        list(extra_summary_logs)
         + parse_logs
         + (
             list(
                 collect_highlander_action_summary_logs(
-                    configs, arguments, configuration_parse_errors
+                    configs,
-                )
+                    arguments,
+                    configuration_parse_errors,
+                ),
             )
             or list(
                 collect_configuration_run_summary_logs(
-                    configs, config_paths, arguments, log_file_path
+                    configs,
-                )
+                    config_paths,
+                    arguments,
+                    log_file_path,
+                ),
             )
         )
     )

borgmatic/commands/completion/actions.py

@@ -22,15 +22,15 @@ def available_actions(subparsers, current_action=None):
     action of "config" but not "list".
     '''
     action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions(
-        subparsers.choices
+        subparsers.choices,
     )
     current_subactions = action_to_subactions.get(current_action)
 
     if current_subactions:
         return current_subactions
 
-    all_subactions = set(
+    all_subactions = {
         subaction for subactions in action_to_subactions.values() for subaction in subactions
-    )
+    }
 
-    return tuple(action for action in subparsers.choices.keys() if action not in all_subactions)
+    return tuple(action for action in subparsers.choices if action not in all_subactions)

borgmatic/commands/completion/bash.py

@@ -23,7 +23,7 @@ def bash_completion():
     borgmatic's command-line argument parsers.
     '''
     (
-        unused_global_parser,
+        _,
         action_parsers,
         global_plus_action_parser,
     ) = borgmatic.commands.arguments.make_parsers(
@@ -33,6 +33,7 @@ def bash_completion():
     global_flags = parser_flags(global_plus_action_parser)
 
     # Avert your eyes.
+    # fmt: off
     return '\n'.join(
         (
             'check_version() {',
@@ -47,24 +48,22 @@ def bash_completion():
             '    fi',
             '}',
             'complete_borgmatic() {',
-        )
+            *tuple(
-        + tuple(
             '''    if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
         COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
         return 0
-    fi'''
+    fi'''  # noqa: UP031
             % (
                 action,
                 parser_flags(action_parser),
                 ' '.join(
-                    borgmatic.commands.completion.actions.available_actions(action_parsers, action)
+                    borgmatic.commands.completion.actions.available_actions(action_parsers, action),
                 ),
                 global_flags,
             )
             for action, action_parser in reversed(action_parsers.choices.items())
-        )
+        ),
-        + (
+            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: UP031
-            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
             % (
                 ' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)),
                 global_flags,
@@ -72,5 +71,5 @@ def bash_completion():
             '    (check_version &)',
             '}',
             '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
-        )
+        ),
     )

borgmatic/commands/completion/fish.py

@@ -11,10 +11,10 @@ def has_file_options(action: Action):
     '''
     Given an argparse.Action instance, return True if it takes a file argument.
     '''
-    return action.metavar in (
+    return action.metavar in {
         'FILENAME',
         'PATH',
-    ) or action.dest in ('config_paths',)
+    } or action.dest in {'config_paths'}
 
 
 def has_choice_options(action: Action):
@@ -36,11 +36,11 @@ def has_unknown_required_param_options(action: Action):
     return (
         action.required is True
         or action.nargs
-        in (
+        in {
             '+',
             '*',
-        )
+        }
-        or action.metavar in ('PATTERN', 'KEYS', 'N')
+        or action.metavar in {'PATTERN', 'KEYS', 'N'}
         or (action.type is not None and action.default is None)
     )
 
@@ -77,7 +77,7 @@ def exact_options_completion(action: Action):
         return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"'''
 
     raise ValueError(
-        f'Unexpected action: {action} passes has_exact_options but has no choices produced'
+        f'Unexpected action: {action} passes has_exact_options but has no choices produced',
     )
 
 
@@ -96,7 +96,7 @@ def fish_completion():
     borgmatic's command-line argument parsers.
     '''
     (
-        unused_global_parser,
+        _,
         action_parsers,
         global_plus_action_parser,
     ) = borgmatic.commands.arguments.make_parsers(
@@ -104,7 +104,7 @@ def fish_completion():
         unparsed_arguments=(),
     )
 
-    all_action_parsers = ' '.join(action for action in action_parsers.choices.keys())
+    all_action_parsers = ' '.join(action for action in action_parsers.choices)
 
     exact_option_args = tuple(
         ' '.join(action.option_strings)
@@ -119,8 +119,9 @@ def fish_completion():
     )
 
     # Avert your eyes.
-    return '\n'.join(
+    # fmt: off
-        dedent_strip_as_tuple(
+    return '\n'.join((
+        *dedent_strip_as_tuple(
             f'''
             function __borgmatic_check_version
                 set -fx this_filename (status current-filename)
@@ -157,27 +158,27 @@ def fish_completion():
 
             set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}"
             set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
-            '''
+            ''',
-        )
+        ),
-        + ('\n# action_parser completions',)
+        '\n# action_parser completions',
-        + tuple(
+        *tuple(
             f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}'''
             for action_name, action_parser in action_parsers.choices.items()
-        )
+        ),
-        + ('\n# global flags',)
+        '\n# global flags',
-        + tuple(
+        *tuple(
             # -n is checked in order, so put faster / more likely to be true checks first
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
             for action in global_plus_action_parser._actions
             # ignore the noargs action, as this is an impossible completion for fish
             if len(action.option_strings) > 0
             if 'Deprecated' not in action.help
-        )
+        ),
-        + ('\n# action_parser flags',)
+        '\n# action_parser flags',
-        + tuple(
+        *tuple(
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
             for action_name, action_parser in action_parsers.choices.items()
             for action in action_parser._actions
             if 'Deprecated' not in (action.help or ())
-        )
+        ),
-    )
+    ))

borgmatic/commands/completion/flag.py

@@ -5,7 +5,7 @@ def variants(flag_name):
     "--foo[9].bar".
     '''
     if '[0]' in flag_name:
-        for index in range(0, 10):
+        for index in range(10):
             yield flag_name.replace('[0]', f'[{index}]')
 
         return

borgmatic/commands/generate_config.py

@@ -10,8 +10,8 @@ def main():
             levelno=logging.WARNING,
             levelname='WARNING',
             msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
-        )
+        ),
     )
 
-    sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
+    sys.argv = ['borgmatic', 'config', 'generate', *sys.argv[1:]]
     borgmatic.commands.borgmatic.main([warning_log])

borgmatic/commands/validate_config.py

@@ -10,8 +10,8 @@ def main():
             levelno=logging.WARNING,
             levelname='WARNING',
             msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.',
-        )
+        ),
     )
 
-    sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:]
+    sys.argv = ['borgmatic', 'config', 'validate', *sys.argv[1:]]
     borgmatic.commands.borgmatic.main([warning_log])

borgmatic/config/arguments.py

@@ -146,16 +146,16 @@ def prepare_arguments_for_config(global_arguments, schema):
         keys = tuple(argument_name.split('.'))
         option_type = type_for_option(schema, keys)
 
-        # The argument doesn't correspond to any option in the schema, so ignore it. It's
+        # The argument doesn't correspond to any option in the schema, or it is a complex argument, so ignore it.
-        # probably a flag that borgmatic has on the command-line but not in configuration.
+        # It's probably a flag that borgmatic has on the command-line but not in configuration.
-        if option_type is None:
+        if option_type in {'object', None}:
             continue
 
         prepared_values.append(
             (
                 keys,
                 convert_value_type(value, option_type),
-            )
+            ),
         )
 
     return tuple(prepared_values)

borgmatic/config/collect.py

@@ -49,6 +49,6 @@ def collect_config_filenames(config_paths):
 
         for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
-            matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
+            matching_filetype = full_filename.endswith(('.yaml', '.yml'))
             if matching_filetype and not os.path.isdir(full_filename):
                 yield os.path.abspath(full_filename)

borgmatic/config/constants.py

@@ -1,3 +1,4 @@
+import contextlib
 import shlex
 
 
@@ -6,17 +7,18 @@ def coerce_scalar(value):
     Given a configuration value, coerce it to an integer or a boolean as appropriate and return the
     result.
     '''
-    try:
+    with contextlib.suppress(TypeError, ValueError):
         return int(value)
-    except (TypeError, ValueError):
-        pass
 
-    if value == 'true' or value == 'True':
+    try:
-        return True
+        return {
-    if value == 'false' or value == 'False':
+            'true': True,
-        return False
+            'True': True,
-
+            'false': False,
-    return value
+            'False': False,
+        }.get(value, value)
+    except TypeError:  # e.g. for an unhashable type
+        return value
 
 
 def apply_constants(value, constants, shell_escape=False):
@@ -55,8 +57,7 @@ def apply_constants(value, constants, shell_escape=False):
                 constants,
                 shell_escape=(
                     shell_escape
-                    or option_name.startswith('before_')
+                    or option_name.startswith(('before_', 'after_'))
-                    or option_name.startswith('after_')
                     or option_name == 'on_error'
                 ),
             )

borgmatic/config/environment.py

@@ -2,7 +2,7 @@ import os
 import re
 
 VARIABLE_PATTERN = re.compile(
-    r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
+    r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})',
 )
 
 

borgmatic/config/generate.py

@@ -1,4 +1,5 @@
 import collections
+import contextlib
 import io
 import os
 import re
@@ -18,7 +19,8 @@ def insert_newline_before_comment(config, field_name):
     field and its comments.
     '''
     config.ca.items[field_name][1].insert(
-        0, ruamel.yaml.tokens.CommentToken('\n', ruamel.yaml.error.CommentMark(0), None)
+        0,
+        ruamel.yaml.tokens.CommentToken('\n', ruamel.yaml.error.CommentMark(0), None),
     )
 
 
@@ -40,18 +42,22 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
         config = ruamel.yaml.comments.CommentedSeq(
             example
             if borgmatic.config.schema.compare_types(
-                schema['items'].get('type'), SCALAR_SCHEMA_TYPES
+                schema['items'].get('type'),
+                SCALAR_SCHEMA_TYPES,
             )
             else [
                 schema_to_sample_configuration(
-                    schema['items'], source_config, level, parent_is_sequence=True
+                    schema['items'],
-                )
+                    source_config,
-            ]
+                    level,
+                    parent_is_sequence=True,
+                ),
+            ],
         )
         add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
     elif borgmatic.config.schema.compare_types(schema_type, {'object'}):
         if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict):
-            source_config = dict(collections.ChainMap(*source_config))
+            source_config = source_config[0]
 
         config = (
             ruamel.yaml.comments.CommentedMap(
@@ -59,19 +65,25 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
                     (
                         field_name,
                         schema_to_sample_configuration(
-                            sub_schema, (source_config or {}).get(field_name, {}), level + 1
+                            sub_schema,
+                            (source_config or {}).get(field_name, {}),
+                            level + 1,
                         ),
                     )
                     for field_name, sub_schema in borgmatic.config.schema.get_properties(
-                        schema
+                        schema,
                     ).items()
-                ]
+                ],
             )
             or example
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
         add_comments_to_configuration_object(
-            config, schema, source_config, indent=indent, skip_first=parent_is_sequence
+            config,
+            schema,
+            source_config,
+            indent=indent,
+            skip_first_field=parent_is_sequence,
         )
     elif borgmatic.config.schema.compare_types(schema_type, SCALAR_SCHEMA_TYPES, match=all):
         return example
@@ -108,17 +120,26 @@ def comment_out_optional_configuration(rendered_config):
     '''
     lines = []
     optional = False
+    indent_characters = None
+    indent_characters_at_sentinel = None
 
     for line in rendered_config.split('\n'):
+        indent_characters = len(line) - len(line.lstrip())
+
         # Upon encountering an optional configuration option, comment out lines until the next blank
         # line.
         if line.strip().startswith(f'# {COMMENTED_OUT_SENTINEL}'):
             optional = True
+            indent_characters_at_sentinel = indent_characters
             continue
 
-        # Hit a blank line, so reset commenting.
+        # Hit a blank line or dedented, so reset commenting.
-        if not line.strip():
+        if not line.strip() or (
+            indent_characters_at_sentinel is not None
+            and indent_characters < indent_characters_at_sentinel
+        ):
             optional = False
+            indent_characters_at_sentinel = None
 
         lines.append(comment_out_line(line) if optional else line)
 
@@ -145,15 +166,13 @@ def write_configuration(config_filename, rendered_config, mode=0o600, overwrite=
     '''
     if not overwrite and os.path.exists(config_filename):
         raise FileExistsError(
-            f'{config_filename} already exists. Aborting. Use --overwrite to replace the file.'
+            f'{config_filename} already exists. Aborting. Use --overwrite to replace the file.',
         )
 
-    try:
+    with contextlib.suppress(FileExistsError, FileNotFoundError):
         os.makedirs(os.path.dirname(config_filename), mode=0o700)
-    except (FileExistsError, FileNotFoundError):
-        pass
 
-    with open(config_filename, 'w') as config_file:
+    with open(config_filename, 'w', encoding='utf-8') as config_file:
         config_file.write(rendered_config)
 
     os.chmod(config_filename, mode)
@@ -178,7 +197,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
     if schema['items'].get('type') != 'object':
         return
 
-    for field_name in config[0].keys():
+    for field_name in config[0]:
         field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {})
         description = field_schema.get('description')
 
@@ -198,32 +217,39 @@ COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
 
 
 def add_comments_to_configuration_object(
-    config, schema, source_config=None, indent=0, skip_first=False
+    config,
+    schema,
+    source_config=None,
+    indent=0,
+    skip_first_field=False,
 ):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
     configuration dict, putting them before each field. Indent the comment the given number of
     characters.
 
+    If skip_first_field is True, omit the comment for the initial field. This is useful for
+    sequences, where the comment for the first field goes before the sequence itself.
+
     And a sentinel for commenting out options that are neither in DEFAULT_KEYS nor the the given
     source configuration dict. The idea is that any options used in the source configuration should
     stay active in the generated configuration.
     '''
     for index, field_name in enumerate(config.keys()):
-        if skip_first and index == 0:
+        if skip_first_field and index == 0:
             continue
 
         field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {})
         description = field_schema.get('description', '').strip()
 
-        # If this isn't a default key, add an indicator to the comment flagging it to be commented
+        # If this isn't a default key, add an indicator to the comment, flagging it to be commented
         # out from the sample configuration. This sentinel is consumed by downstream processing that
         # does the actual commenting out.
         if field_name not in DEFAULT_KEYS and (
             source_config is None or field_name not in source_config
         ):
             description = (
-                '\n'.join((description, COMMENTED_OUT_SENTINEL))
+                f'{description}\n{COMMENTED_OUT_SENTINEL}'
                 if description
                 else COMMENTED_OUT_SENTINEL
             )
@@ -251,6 +277,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
     '''
     if not source_config:
         return destination_config
+
     if not destination_config or not isinstance(source_config, collections.abc.Mapping):
         return source_config
 
@@ -258,7 +285,8 @@ def merge_source_configuration_into_destination(destination_config, source_confi
         # This is a mapping. Recurse for this key/value.
         if isinstance(source_value, collections.abc.Mapping):
             destination_config[field_name] = merge_source_configuration_into_destination(
-                destination_config[field_name], source_value
+                destination_config[field_name],
+                source_value,
             )
             continue
 
@@ -272,18 +300,22 @@ def merge_source_configuration_into_destination(destination_config, source_confi
                         source_item,
                     )
                     for index, source_item in enumerate(source_value)
-                ]
+                ],
             )
             continue
 
         # This is some sort of scalar. Set it into the destination.
-        destination_config[field_name] = source_config[field_name]
+        destination_config[field_name] = source_value
 
     return destination_config
 
 
 def generate_sample_configuration(
-    dry_run, source_filename, destination_filename, schema_filename, overwrite=False
+    dry_run,
+    source_filename,
+    destination_filename,
+    schema_filename,
+    overwrite=False,
 ):
     '''
     Given an optional source configuration filename, and a required destination configuration
@@ -292,15 +324,22 @@ def generate_sample_configuration(
     schema. If a source filename is provided, merge the parsed contents of that configuration into
     the generated configuration.
     '''
-    schema = ruamel.yaml.YAML(typ='safe').load(open(schema_filename))
+    schema = ruamel.yaml.YAML(typ='safe').load(open(schema_filename, encoding='utf-8'))
     source_config = None
 
     if source_filename:
         source_config = load.load_configuration(source_filename)
         normalize.normalize(source_filename, source_config)
 
+        # The borgmatic.config.normalize.normalize() function tacks on an empty "bootstrap" if
+        # needed, so the hook gets used by default. But we don't want it to end up in the generated
+        # config unless the user has set it explicitly, as an empty "bootstrap:" won't validate.
+        if source_config and source_config.get('bootstrap') == {}:
+            del source_config['bootstrap']
+
     destination_config = merge_source_configuration_into_destination(
-        schema_to_sample_configuration(schema, source_config), source_config
+        schema_to_sample_configuration(schema, source_config),
+        source_config,
     )
 
     if dry_run:

borgmatic/config/load.py

@@ -31,7 +31,7 @@ def probe_and_include_file(filename, include_directories, config_paths):
             return load_configuration(candidate_filename, config_paths)
 
     raise FileNotFoundError(
-        f'Could not find include {filename} at {" or ".join(candidate_filenames)}'
+        f'Could not find include {filename} at {" or ".join(candidate_filenames)}',
     )
 
 
@@ -69,7 +69,7 @@ def include_configuration(loader, filename_node, include_directory, config_paths
         ]
 
     raise ValueError(
-        'The value given for the !include tag is invalid; use a single filename or a list of filenames instead'
+        'The value given for the !include tag is invalid; use a single filename or a list of filenames instead',
     )
 
 
@@ -85,7 +85,7 @@ def raise_retain_node_error(loader, node):
     '''
     if isinstance(node, (ruamel.yaml.nodes.MappingNode, ruamel.yaml.nodes.SequenceNode)):
         raise ValueError(
-            'The !retain tag may only be used within a configuration file containing a merged !include tag.'
+            'The !retain tag may only be used within a configuration file containing a merged !include tag.',
         )
 
     raise ValueError('The !retain tag may only be used on a mapping or list.')
@@ -100,7 +100,7 @@ def raise_omit_node_error(loader, node):
     tags are handled by deep_merge_nodes() below.
     '''
     raise ValueError(
-        'The !omit tag may only be used on a scalar (e.g., string) or list element within a configuration file containing a merged !include tag.'
+        'The !omit tag may only be used on a scalar (e.g., string) or list element within a configuration file containing a merged !include tag.',
     )
 
 
@@ -111,9 +111,13 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
 
     def __init__(
-        self, preserve_quotes=None, loader=None, include_directory=None, config_paths=None
+        self,
+        preserve_quotes=None,
+        loader=None,
+        include_directory=None,
+        config_paths=None,
     ):
-        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        super().__init__(preserve_quotes, loader)
         self.add_constructor(
             '!include',
             functools.partial(
@@ -147,7 +151,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
         representer = ruamel.yaml.representer.SafeRepresenter()
 
         for index, (key_node, value_node) in enumerate(node.value):
-            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+            if key_node.tag == 'tag:yaml.org,2002:merge' and value_node.tag == '!include':
                 # Replace the merge include with a sequence of included configuration nodes ready
                 # for merging. The construct_object() call here triggers include_configuration()
                 # among other constructors.
@@ -157,7 +161,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
                 )
 
         # This super().flatten_mapping() call actually performs "<<" merges.
-        super(Include_constructor, self).flatten_mapping(node)
+        super().flatten_mapping(node)
 
         node.value = deep_merge_nodes(node.value)
 
@@ -179,7 +183,7 @@ def load_configuration(filename, config_paths=None):
     # because yaml.Constructor has to be an actual class.)
     class Include_constructor_with_extras(Include_constructor):
         def __init__(self, preserve_quotes=None, loader=None):
-            super(Include_constructor_with_extras, self).__init__(
+            super().__init__(
                 preserve_quotes,
                 loader,
                 include_directory=os.path.dirname(filename),
@@ -190,7 +194,7 @@ def load_configuration(filename, config_paths=None):
     yaml.Constructor = Include_constructor_with_extras
     config_paths.add(filename)
 
-    with open(filename) as file:
+    with open(filename, encoding='utf-8') as file:
         return yaml.load(file.read())
 
 
@@ -318,17 +322,18 @@ def deep_merge_nodes(nodes):
 
     # Bucket the nodes by their keys. Then merge all of the values sharing the same key.
     for key_name, grouped_nodes in itertools.groupby(
-        sorted(nodes, key=get_node_key_name), get_node_key_name
+        sorted(nodes, key=get_node_key_name),
+        get_node_key_name,
     ):
-        grouped_nodes = list(grouped_nodes)
+        grouped_nodes = list(grouped_nodes)  # noqa: PLW2901
 
         # The merged node inherits its attributes from the final node in the group.
         (last_node_key, last_node_value) = grouped_nodes[-1]
-        value_types = set(type(value) for (_, value) in grouped_nodes)
+        value_types = {type(value) for (_, value) in grouped_nodes}
 
         if len(value_types) > 1:
             raise ValueError(
-                f'Incompatible types found when trying to merge "{key_name}:" values across configuration files: {", ".join(value_type.id for value_type in value_types)}'
+                f'Incompatible types found when trying to merge "{key_name}:" values across configuration files: {", ".join(value_type.id for value_type in value_types)}',
             )
 
         # If we're dealing with MappingNodes, recurse and merge its values as well.
@@ -351,7 +356,7 @@ def deep_merge_nodes(nodes):
                             comment=last_node_value.comment,
                             anchor=last_node_value.anchor,
                         ),
-                    )
+                    ),
                 )
 
             continue
@@ -374,7 +379,7 @@ def deep_merge_nodes(nodes):
                             comment=last_node_value.comment,
                             anchor=last_node_value.anchor,
                         ),
-                    )
+                    ),
                 )
 
             continue

borgmatic/config/normalize.py

@@ -25,12 +25,12 @@ def normalize_sections(config_filename, config):
         and location.get('prefix') != consistency.get('prefix')
     ):
         raise ValueError(
-            'The retention prefix and the consistency prefix cannot have different values (unless one is not set).'
+            'The retention prefix and the consistency prefix cannot have different values (unless one is not set).',
         )
 
     if storage.get('umask') and hooks.get('umask') and storage.get('umask') != hooks.get('umask'):
         raise ValueError(
-            'The storage umask and the hooks umask cannot have different values (unless one is not set).'
+            'The storage umask and the hooks umask cannot have different values (unless one is not set).',
         )
 
     any_section_upgraded = False
@@ -51,8 +51,8 @@ def normalize_sections(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: Configuration sections (like location:, storage:, retention:, consistency:, and hooks:) are deprecated and support will be removed from a future release. To prepare for this, move your options out of sections to the global scope.',
-                )
+                ),
-            )
+            ),
         ]
 
     return []
@@ -68,7 +68,7 @@ def make_command_hook_deprecation_log(config_filename, option_name):  # pragma:
             levelno=logging.WARNING,
             levelname='WARNING',
             msg=f'{config_filename}: {option_name} is deprecated and support will be removed from a future release. Use commands: instead.',
-        )
+        ),
     )
 
 
@@ -90,7 +90,7 @@ def normalize_commands(config_filename, config):
                 {
                     preposition: 'repository',
                     'run': commands,
-                }
+                },
             )
 
     # Normalize "before_backup", "before_prune", "after_backup", "after_prune", etc.
@@ -108,7 +108,7 @@ def normalize_commands(config_filename, config):
                     preposition: 'action',
                     'when': [action_name],
                     'run': commands,
-                }
+                },
             )
 
     # Normalize "on_error".
@@ -121,7 +121,7 @@ def normalize_commands(config_filename, config):
                 'after': 'error',
                 'when': ['create', 'prune', 'compact', 'check'],
                 'run': commands,
-            }
+            },
         )
 
     # Normalize "before_everything" and "after_everything".
@@ -136,13 +136,13 @@ def normalize_commands(config_filename, config):
                     preposition: 'everything',
                     'when': ['create'],
                     'run': commands,
-                }
+                },
             )
 
     return logs
 
 
-def normalize(config_filename, config):
+def normalize(config_filename, config):  # noqa: PLR0912, PLR0915
     '''
     Given a configuration filename and a configuration dict of its loaded contents, apply particular
     hard-coded rules to normalize the configuration to adhere to the current schema. Return any log
@@ -159,9 +159,9 @@ def normalize(config_filename, config):
                 dict(
                     levelno=logging.WARNING,
                     levelname='WARNING',
-                    msg=f'{config_filename}: The borgmatic_source_directory option is deprecated and will be removed from a future release. Use borgmatic_runtime_directory and borgmatic_state_directory instead.',
+                    msg=f'{config_filename}: The borgmatic_source_directory option is deprecated and will be removed from a future release. Use user_runtime_directory and user_state_directory instead.',
-                )
+                ),
-            )
+            ),
         )
 
     # Upgrade exclude_if_present from a string to a list.
@@ -173,8 +173,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The exclude_if_present option now expects a list value. String values for this option are deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['exclude_if_present'] = [exclude_if_present]
 
@@ -191,8 +191,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The store_config_files option has moved under the bootstrap hook. Specifying store_config_files at the global scope is deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         del config['store_config_files']
         config['bootstrap']['store_config_files'] = store_config_files
@@ -206,8 +206,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects a key/value pair with "ping_url" as a key. String values for this option are deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['healthchecks'] = {'ping_url': healthchecks}
 
@@ -219,8 +219,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['cronitor'] = {'ping_url': cronitor}
 
@@ -232,8 +232,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['pagerduty'] = {'integration_key': pagerduty}
 
@@ -245,8 +245,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The healthchecks hook now expects key/value pairs. String values for this option are deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['cronhub'] = {'ping_url': cronhub}
 
@@ -259,8 +259,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The checks option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['checks'] = [{'name': check_type} for check_type in checks]
 
@@ -273,8 +273,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The numeric_owner option has been renamed to numeric_ids. numeric_owner is deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['numeric_ids'] = numeric_owner
 
@@ -286,8 +286,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The bsd_flags option has been renamed to flags. bsd_flags is deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['flags'] = bsd_flags
 
@@ -299,8 +299,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The remote_rate_limit option has been renamed to upload_rate_limit. remote_rate_limit is deprecated and support will be removed from a future release.',
-                )
+                ),
-            )
+            ),
         )
         config['upload_rate_limit'] = remote_rate_limit
 
@@ -314,8 +314,8 @@ def normalize(config_filename, config):
                         levelno=logging.WARNING,
                         levelname='WARNING',
                         msg=f'{config_filename}: The repositories option now expects a list of key/value pairs. Lists of strings for this option are deprecated and support will be removed from a future release.',
-                    )
+                    ),
-                )
+                ),
             )
             config['repositories'] = [
                 {'path': repository} if isinstance(repository, str) else repository
@@ -338,25 +338,22 @@ def normalize(config_filename, config):
                             levelno=logging.WARNING,
                             levelname='WARNING',
                             msg=f'{config_filename}: Repository paths containing "~" are deprecated in borgmatic and support will be removed from a future release.',
-                        )
+                        ),
-                    )
+                    ),
                 )
+
             if ':' in repository_path:
                 if repository_path.startswith('file://'):
                     updated_repository_path = os.path.abspath(
-                        repository_path.partition('file://')[-1]
+                        repository_path.partition('file://')[-1],
                     )
                     config['repositories'].append(
                         dict(
                             repository_dict,
                             path=updated_repository_path,
-                        )
+                        ),
                     )
-                elif (
+                elif repository_path.startswith(('ssh://', 'sftp://', 'rclone:', 's3:', 'b2:')):
-                    repository_path.startswith('ssh://')
-                    or repository_path.startswith('sftp://')
-                    or repository_path.startswith('rclone:')
-                ):
                     config['repositories'].append(repository_dict)
                 else:
                     rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
@@ -365,15 +362,15 @@ def normalize(config_filename, config):
                             dict(
                                 levelno=logging.WARNING,
                                 levelname='WARNING',
-                                msg=f'{config_filename}: Remote repository paths without ssh:// or rclone: syntax are deprecated and support will be removed from a future release. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
+                                msg=f'{config_filename}: Remote repository paths without ssh://, sftp://, rclone:, s3:, or b2:, syntax are deprecated and support will be removed from a future release. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
-                            )
+                            ),
-                        )
+                        ),
                     )
                     config['repositories'].append(
                         dict(
                             repository_dict,
                             path=rewritten_repository_path,
-                        )
+                        ),
                     )
             else:
                 config['repositories'].append(repository_dict)
@@ -385,8 +382,8 @@ def normalize(config_filename, config):
                     levelno=logging.WARNING,
                     levelname='WARNING',
                     msg=f'{config_filename}: The prefix option is deprecated and support will be removed from a future release. Use archive_name_format or match_archives instead.',
-                )
+                ),
-            )
+            ),
         )
 
     return logs

borgmatic/config/override.py

@@ -18,7 +18,7 @@ def set_values(config, keys, value):
     if len(keys) == 1:
         if isinstance(config, list):
             raise ValueError(
-                'When overriding a list option, the value must use list syntax (e.g., "[foo, bar]" or "[{key: value}]" as appropriate)'
+                'When overriding a list option, the value must use list syntax (e.g., "[foo, bar]" or "[{key: value}]" as appropriate)',
             )
 
         config[first_key] = value
@@ -69,11 +69,11 @@ def type_for_option(schema, option_keys):
     '''
     option_schema = schema
 
-    for key in option_keys:
+    try:
-        try:
+        for key in option_keys:
             option_schema = option_schema['properties'][key]
-        except KeyError:
+    except KeyError:
-            return None
+        return None
 
     try:
         return option_schema['type']
@@ -103,8 +103,8 @@ def parse_overrides(raw_overrides, schema):
 
     parsed_overrides = []
 
-    for raw_override in raw_overrides:
+    try:
-        try:
+        for raw_override in raw_overrides:
             raw_keys, value = raw_override.split('=', 1)
             keys = tuple(raw_keys.split('.'))
             option_type = type_for_option(schema, keys)
@@ -113,14 +113,14 @@ def parse_overrides(raw_overrides, schema):
                 (
                     keys,
                     convert_value_type(value, option_type),
-                )
+                ),
             )
-        except ValueError:
+    except ValueError:
-            raise ValueError(
+        raise ValueError(
-                f"Invalid override '{raw_override}'. Make sure you use the form: OPTION=VALUE or OPTION.SUBOPTION=VALUE"
+            f"Invalid override '{raw_override}'. Make sure you use the form: OPTION=VALUE or OPTION.SUBOPTION=VALUE",
-            )
+        )
-        except ruamel.yaml.error.YAMLError as error:
+    except ruamel.yaml.error.YAMLError as error:
-            raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
+        raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
 
     return tuple(parsed_overrides)
 
@@ -139,7 +139,7 @@ def apply_overrides(config, schema, raw_overrides):
 
     if overrides:
         logger.warning(
-            "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set."
+            "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set.",
         )
 
     for keys, value in overrides:

borgmatic/config/paths.py

@@ -1,3 +1,4 @@
+import contextlib
 import logging
 import os
 import tempfile
@@ -34,7 +35,8 @@ TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-'
 
 
 def replace_temporary_subdirectory_with_glob(
-    path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX
+    path,
+    temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX,
 ):
     '''
     Given an absolute temporary directory path and an optional temporary directory prefix, look for
@@ -103,7 +105,9 @@ class Runtime_directory:
 
             self.temporary_directory = None
         else:
-            base_directory = os.environ.get('TMPDIR') or os.environ.get('TEMP') or '/tmp'
+            base_directory = (
+                os.environ.get('TMPDIR') or os.environ.get('TEMP') or '/tmp'  # noqa: S108
+            )
 
             if not base_directory.startswith(os.path.sep):
                 raise ValueError('The temporary directory must be an absolute path')
@@ -122,7 +126,7 @@ class Runtime_directory:
                 base_path if final_directory == 'borgmatic' else runtime_directory,
                 '.',  # Borg 1.4+ "slashdot" hack.
                 'borgmatic',
-            )
+            ),
         )
         os.makedirs(self.runtime_path, mode=0o700, exist_ok=True)
 
@@ -139,13 +143,11 @@ class Runtime_directory:
         Delete any temporary directory that was created as part of initialization.
         '''
         if self.temporary_directory:
-            try:
-                self.temporary_directory.cleanup()
             # The cleanup() call errors if, for instance, there's still a
             # mounted filesystem within the temporary directory. There's
             # nothing we can do about that here, so swallow the error.
-            except OSError:
+            with contextlib.suppress(OSError):
-                pass
+                self.temporary_directory.cleanup()
 
 
 def make_runtime_directory_glob(borgmatic_runtime_directory):
@@ -158,7 +160,7 @@ def make_runtime_directory_glob(borgmatic_runtime_directory):
         *(
             '*' if subdirectory.startswith(TEMPORARY_DIRECTORY_PREFIX) else subdirectory
             for subdirectory in os.path.normpath(borgmatic_runtime_directory).split(os.path.sep)
-        )
+        ),
     )
 
 
@@ -175,5 +177,5 @@ def get_borgmatic_state_directory(config):
             or os.environ.get('STATE_DIRECTORY')  # Set by systemd if configured.
             or '~/.local/state',
             'borgmatic',
-        )
+        ),
     )

borgmatic/config/schema.py

@@ -14,8 +14,8 @@ def get_properties(schema):
             item
             for item in itertools.chain(
                 *itertools.zip_longest(
-                    *[sub_schema['properties'].items() for sub_schema in schema['oneOf']]
+                    *[sub_schema['properties'].items() for sub_schema in schema['oneOf']],
-                )
+                ),
             )
             if item is not None
         )
@@ -61,12 +61,6 @@ def compare_types(schema_type, target_types, match=any):
     list must be in the target types.
     '''
     if isinstance(schema_type, list):
-        if match(element_schema_type in target_types for element_schema_type in schema_type):
+        return match(element_schema_type in target_types for element_schema_type in schema_type)
-            return True
 
-        return False
+    return schema_type in target_types
-
-    if schema_type in target_types:
-        return True
-
-    return False

borgmatic/config/schema.yaml

@@ -72,7 +72,8 @@ properties:
                     description: |
                         Whether any missing parent directories of the repository
                         path should be created, only used for the repo-create
-                        action. Defaults to false.
+                        action. Defaults to false. (This option is supported 
+                        for Borg 1.x only)
                     example: true
         description: |
             A required list of local or remote repositories with paths and
@@ -389,6 +390,13 @@ properties:
             Path for Borg cache files. Defaults to
             $borg_base_directory/.cache/borg
         example: /path/to/base/cache
+    use_chunks_archive:
+        type: boolean
+        description: |
+            Enables or disables the use of chunks.archive.d for faster cache
+            resyncs in Borg. If true, value is set to "yes" (default) else
+            it's set to "no", reducing disk usage but slowing resyncs.
+        example: true
     borg_files_cache_ttl:
         type: integer
         description: |
@@ -483,6 +491,19 @@ properties:
             Bypass Borg error about a previously unknown unencrypted repository.
             Defaults to false.
         example: true
+    debug_passphrase:
+        type: boolean
+        description: |
+            When set true, display debugging information that includes 
+            passphrases used and passphrase related environment variables set. 
+            Defaults to false.
+        example: true
+    display_passphrase:
+        type: boolean
+        description: |
+            When set true, always shows passphrase and its hex UTF-8 byte
+            sequence. Defaults to false.
+        example: true
     check_i_know_what_i_am_doing:
         type: boolean
         description: |
@@ -558,6 +579,14 @@ properties:
         type: integer
         description: Number of yearly archives to keep.
         example: 1
+    keep_13weekly:
+        type: integer
+        description: Number of quarterly archives to keep (13 week strategy).
+        example: 13
+    keep_3monthly:
+        type: integer
+        description: Number of quarterly archives to keep (3 month strategy).
+        example: 3
     prefix:
         type: string
         description: |
@@ -1111,6 +1140,7 @@ properties:
                       before:
                           type: string
                           enum:
+                              - step
                               - action
                               - repository
                               - configuration
@@ -1119,6 +1149,8 @@ properties:
                               Name for the point in borgmatic's execution that
                               the commands should be run before (required if
                               "after" isn't set):
+                               * "step" runs before a sub-action step for each
+                              repository, e.g. for an individual check.
                                * "action" runs before each action for each
                               repository.
                                * "repository" runs before all actions for each
@@ -1159,6 +1191,18 @@ properties:
                               List of actions for which the commands will be
                               run. Defaults to running for all actions.
                           example: [create, prune, compact, check]
+                      steps:
+                          type: array
+                          items:
+                              type: string
+                              enum:
+                                  - archives_repository_data
+                                  - extract
+                                  - spot
+                          description: |
+                              List of sub-action steps for which the commands
+                              will be run. Defaults to running for all steps.
+                          example: [extract, spot]
                       run:
                           type: array
                           items:
@@ -1174,6 +1218,7 @@ properties:
                       after:
                           type: string
                           enum:
+                              - step
                               - action
                               - repository
                               - configuration
@@ -1183,6 +1228,8 @@ properties:
                               Name for the point in borgmatic's execution that
                               the commands should be run after (required if
                               "before" isn't set):
+                               * "step" runs before a sub-action step for each
+                              repository, e.g. for an individual check.
                                * "action" runs after each action for each
                               repository.
                                * "repository" runs after all actions for each
@@ -1225,6 +1272,38 @@ properties:
                               particular actions listed here. Defaults to
                               running for all actions.
                           example: [create, prune, compact, check]
+                      steps:
+                          type: array
+                          items:
+                              type: string
+                              enum:
+                                  - archives_repository_data
+                                  - extract
+                                  - spot
+                          description: |
+                              List of sub-action steps for which the commands
+                              will be run. Defaults to running for all steps.
+                          example: [extract, spot]
+                      states:
+                          type: array
+                          items:
+                              type: string
+                              enum:
+                                  - finish
+                                  - fail
+                          description: |
+                              Only trigger the hook if borgmatic encounters one
+                              of the states (execution results) listed here,
+                              where:
+                               * "finish": No errors occurred.
+                               * "fail": An error occurred.
+                              This state is evaluated only for the scope of the
+                              configured "action", "repository", etc., rather
+                              than for the entire borgmatic run. Only available
+                              for "after" hooks. Defaults to running the hook
+                              for all states.
+                          example:
+                              - finish
                       run:
                           type: array
                           items:
@@ -1274,6 +1353,23 @@ properties:
                         implicitly enables read_special (see above) to support
                         dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1475,6 +1571,23 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1525,6 +1638,23 @@ properties:
                         Defaults to the "password" option. Supports the
                         "{credential ...}" syntax.
                     example: trustsome1
+                password_transport:
+                    type: string
+                    enum:
+                        - pipe
+                        - environment
+                    description: |
+                        How to transmit database passwords from borgmatic to the
+                        MariaDB client, one of:
+                         * "pipe": Securely transmit passwords via anonymous
+                           pipe. Only works if the database client is on the
+                           same host as borgmatic. (The server can be
+                           somewhere else.) This is the default value.
+                         * "environment": Transmit passwords via environment
+                           variable. Potentially less secure than a pipe, but
+                           necessary when the database client is elsewhere, e.g.
+                           when "mariadb_dump_command" is configured to "exec"
+                           into a container and run a client there.
                 tls:
                     type: boolean
                     description: |
@@ -1623,6 +1753,23 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1673,6 +1820,23 @@ properties:
                         Defaults to the "password" option. Supports the
                         "{credential ...}" syntax.
                     example: trustsome1
+                password_transport:
+                    type: string
+                    enum:
+                        - pipe
+                        - environment
+                    description: |
+                        How to transmit database passwords from borgmatic to the
+                        MySQL client, one of:
+                         * "pipe": Securely transmit passwords via anonymous
+                           pipe. Only works if the database client is on the
+                           same host as borgmatic. (The server can be
+                           somewhere else.) This is the default value.
+                         * "environment": Transmit passwords via environment
+                           variable. Potentially less secure than a pipe, but
+                           necessary when the database client is elsewhere, e.g.
+                           when "mysql_dump_command" is configured to "exec"
+                           into a container and run a client there.
                 tls:
                     type: boolean
                     description: |
@@ -1779,6 +1943,11 @@ properties:
                         read_special (see above) to support dump and restore
                         streaming.
                     example: /var/lib/sqlite/users.db
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
                 restore_path:
                     type: string
                     description: |
@@ -1827,6 +1996,23 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -2846,6 +3032,25 @@ properties:
         description: |
             Configuration for integration with Linux LVM (Logical Volume
             Manager).
+    systemd:
+        type: object
+        additionalProperties: false
+        properties:
+            systemd_creds_command:
+                type: string
+                description: |
+                    Command to use instead of "systemd-creds". Only used as a
+                    fallback when borgmatic is run outside of a systemd service.
+                example: /usr/local/bin/systemd-creds
+            encrypted_credentials_directory:
+                type: string
+                description: |
+                    Directory containing encrypted credentials for
+                    "systemd-creds" to use instead of
+                    "/etc/credstore.encrypted".
+                example: /path/to/credstore.encrypted
+        description: |
+            Configuration for integration with systemd credentials.
     container:
         type: object
         additionalProperties: false

borgmatic/config/validate.py

@@ -17,7 +17,7 @@ def schema_filename():
     '''
     schema_path = os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml')
 
-    with open(schema_path):
+    with open(schema_path, encoding='utf-8'):
         return schema_path
 
 
@@ -97,7 +97,11 @@ def apply_logical_validation(config_filename, parsed_configuration):
 
 
 def parse_configuration(
-    config_filename, schema_filename, arguments, overrides=None, resolve_env=True
+    config_filename,
+    schema_filename,
+    arguments,
+    overrides=None,
+    resolve_env=True,
 ):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
@@ -147,7 +151,8 @@ def parse_configuration(
 
     if validation_errors:
         raise Validation_error(
-            config_filename, tuple(format_json_error(error) for error in validation_errors)
+            config_filename,
+            tuple(format_json_error(error) for error in validation_errors),
         )
 
     apply_logical_validation(config_filename, config)
@@ -166,13 +171,14 @@ def normalize_repository_path(repository, base=None):
         return (
             os.path.abspath(os.path.join(base, repository)) if base else os.path.abspath(repository)
         )
-    elif repository.startswith('file://'):
+
+    if repository.startswith('file://'):
         local_path = repository.partition('file://')[-1]
         return (
             os.path.abspath(os.path.join(base, local_path)) if base else os.path.abspath(local_path)
         )
-    else:
+
-        return repository
+    return repository
 
 
 def glob_match(first, second):
@@ -194,11 +200,13 @@ def repositories_match(first, second):
     '''
     if isinstance(first, str):
         first = {'path': first, 'label': first}
+
     if isinstance(second, str):
         second = {'path': second, 'label': second}
 
     return glob_match(first.get('label'), second.get('label')) or glob_match(
-        normalize_repository_path(first.get('path')), normalize_repository_path(second.get('path'))
+        normalize_repository_path(first.get('path')),
+        normalize_repository_path(second.get('path')),
     )
 
 
@@ -219,7 +227,7 @@ def guard_configuration_contains_repository(repository, configurations):
             for config in configurations.values()
             for config_repository in config['repositories']
             if repositories_match(config_repository, repository)
-        )
+        ),
     )
 
     if count == 0:

borgmatic/execute.py

@@ -31,6 +31,7 @@ def interpret_exit_code(command, exit_code, borg_local_path=None, borg_exit_code
     '''
     if exit_code is None:
         return Exit_status.STILL_RUNNING
+
     if exit_code == 0:
         return Exit_status.SUCCESS
 
@@ -42,12 +43,13 @@ def interpret_exit_code(command, exit_code, borg_local_path=None, borg_exit_code
 
                 if treat_as == 'error':
                     logger.error(
-                        f'Treating exit code {exit_code} as an error, as per configuration'
+                        f'Treating exit code {exit_code} as an error, as per configuration',
                     )
                     return Exit_status.ERROR
-                elif treat_as == 'warning':
+
+                if treat_as == 'warning':
                     logger.warning(
-                        f'Treating exit code {exit_code} as a warning, as per configuration'
+                        f'Treating exit code {exit_code} as a warning, as per configuration',
                     )
                     return Exit_status.WARNING
 
@@ -102,7 +104,7 @@ def append_last_lines(last_lines, captured_output, line, output_log_level):
         logger.log(output_log_level, line)
 
 
-def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes):
+def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes):  # noqa: PLR0912
     '''
     Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
     process with the requested log level. Additionally, raise a CalledProcessError if a process
@@ -131,7 +133,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
     still_running = True
 
     # Log output for each process until they all exit.
-    while True:
+    while True:  # noqa: PLR1702
         if output_buffers:
             (ready_buffers, _, _) = select.select(output_buffers, [], [])
 
@@ -181,7 +183,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
             command = process.args.split(' ') if isinstance(process.args, str) else process.args
             exit_status = interpret_exit_code(command, exit_code, borg_local_path, borg_exit_codes)
 
-            if exit_status in (Exit_status.ERROR, Exit_status.WARNING):
+            if exit_status in {Exit_status.ERROR, Exit_status.WARNING}:
                 # If an error occurs, include its output in the raised exception so that we don't
                 # inadvertently hide error output.
                 output_buffer = output_buffer_for_process(process, exclude_stdouts)
@@ -194,7 +196,10 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
                         break
 
                     append_last_lines(
-                        last_lines, captured_outputs[process], line, output_log_level=logging.ERROR
+                        last_lines,
+                        captured_outputs[process],
+                        line,
+                        output_log_level=logging.ERROR,
                     )
 
                 if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
@@ -209,7 +214,9 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
 
                 if exit_status == Exit_status.ERROR:
                     raise subprocess.CalledProcessError(
-                        exit_code, command_for_process(process), '\n'.join(last_lines)
+                        exit_code,
+                        command_for_process(process),
+                        '\n'.join(last_lines),
                     )
 
                 still_running = False
@@ -220,6 +227,8 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
             process: '\n'.join(output_lines) for process, output_lines in captured_outputs.items()
         }
 
+    return None
+
 
 SECRET_COMMAND_FLAG_NAMES = {'--password'}
 
@@ -255,19 +264,19 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non
             ' '.join(
                 tuple(
                     f'{key}=***'
-                    for key in (environment or {}).keys()
+                    for key in (environment or {})
                     if any(
                         key.startswith(prefix)
                         for prefix in PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG
                     )
                 )
-                + mask_command_secrets(full_command)
+                + mask_command_secrets(full_command),
             ),
             width=MAX_LOGGED_COMMAND_LENGTH,
             placeholder=' ...',
         )
         + (f" < {getattr(input_file, 'name', input_file)}" if input_file else '')
-        + (f" > {getattr(output_file, 'name', output_file)}" if output_file else '')
+        + (f" > {getattr(output_file, 'name', output_file)}" if output_file else ''),
     )
 
 
@@ -288,6 +297,7 @@ def execute_command(
     borg_local_path=None,
     borg_exit_codes=None,
     run_to_completion=True,
+    close_fds=False,  # Necessary for passing credentials via anonymous pipe.
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
@@ -307,7 +317,7 @@ def execute_command(
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
-    process = subprocess.Popen(
+    process = subprocess.Popen(  # noqa: S603
         command,
         stdin=input_file,
         stdout=None if do_not_capture else (output_file or subprocess.PIPE),
@@ -315,8 +325,7 @@ def execute_command(
         shell=shell,
         env=environment,
         cwd=working_directory,
-        # Necessary for passing credentials via anonymous pipe.
+        close_fds=close_fds,
-        close_fds=False,
     )
     if not run_to_completion:
         return process
@@ -330,6 +339,8 @@ def execute_command(
             borg_exit_codes,
         )
 
+    return None
+
 
 def execute_command_and_capture_output(
     full_command,
@@ -340,6 +351,7 @@ def execute_command_and_capture_output(
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
+    close_fds=False,  # Necessary for passing credentials via anonymous pipe.
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its
@@ -358,15 +370,14 @@ def execute_command_and_capture_output(
     command = ' '.join(full_command) if shell else full_command
 
     try:
-        output = subprocess.check_output(
+        output = subprocess.check_output(  # noqa: S603
             command,
             stdin=input_file,
             stderr=subprocess.STDOUT if capture_stderr else None,
             shell=shell,
             env=environment,
             cwd=working_directory,
-            # Necessary for passing credentials via anonymous pipe.
+            close_fds=close_fds,
-            close_fds=False,
         )
     except subprocess.CalledProcessError as error:
         if (
@@ -374,6 +385,7 @@ def execute_command_and_capture_output(
             == Exit_status.ERROR
         ):
             raise
+
         output = error.output
 
     return output.decode() if output is not None else None
@@ -390,6 +402,7 @@ def execute_command_with_processes(
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
+    close_fds=False,  # Necessary for passing credentials via anonymous pipe.
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
@@ -415,7 +428,7 @@ def execute_command_with_processes(
     command = ' '.join(full_command) if shell else full_command
 
     try:
-        command_process = subprocess.Popen(
+        command_process = subprocess.Popen(  # noqa: S603
             command,
             stdin=input_file,
             stdout=None if do_not_capture else (output_file or subprocess.PIPE),
@@ -425,8 +438,7 @@ def execute_command_with_processes(
             shell=shell,
             env=environment,
             cwd=working_directory,
-            # Necessary for passing credentials via anonymous pipe.
+            close_fds=close_fds,
-            close_fds=False,
         )
     except (subprocess.CalledProcessError, OSError):
         # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
@@ -435,11 +447,12 @@ def execute_command_with_processes(
             if process.poll() is None:
                 process.stdout.read(0)
                 process.kill()
+
         raise
 
     with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
         captured_outputs = log_outputs(
-            tuple(processes) + (command_process,),
+            (*processes, command_process),
             (input_file, output_file),
             output_log_level,
             borg_local_path,
@@ -448,3 +461,5 @@ def execute_command_with_processes(
 
     if output_log_level is None:
         return captured_outputs.get(command_process)
+
+    return None

borgmatic/hooks/command.py

@@ -13,6 +13,20 @@ logger = logging.getLogger(__name__)
 
 
 SOFT_FAIL_EXIT_CODE = 75
+BORG_PLACEHOLDERS = {
+    '{hostname}',
+    '{fqdn}',
+    '{reverse-fqdn}',
+    '{now}',
+    '{utcnow}',
+    '{unixtime}',
+    '{user}',
+    '{pid}',
+    '{borgversion}',
+    '{borgmajor}',
+    '{borgminor}',
+    '{borgpatch}',
+}
 
 
 def interpolate_context(hook_description, command, context):
@@ -23,10 +37,13 @@ def interpolate_context(hook_description, command, context):
     for name, value in context.items():
         command = command.replace(f'{{{name}}}', shlex.quote(str(value)))
 
-    for unsupported_variable in re.findall(r'{\w+}', command):
+    for unsupported_variable in re.findall(r'\{\w+\}', command):
-        logger.warning(
+        # Warn about variables unknown to borgmatic, but don't warn if the variable name happens to
-            f"Variable '{unsupported_variable}' is not supported in {hook_description} hook"
+        # be a Borg placeholder, as Borg should hopefully consume it.
-        )
+        if unsupported_variable not in BORG_PLACEHOLDERS:
+            logger.warning(
+                f'Variable "{unsupported_variable}" is not supported in the {hook_description} hook',
+            )
 
     return command
 
@@ -47,25 +64,34 @@ def make_environment(current_environment, sys_module=sys):
     return environment
 
 
-def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_names=None):
+def filter_hooks(command_hooks, before=None, after=None, action_names=None, step_names=None, state_names=None):
     '''
     Given a sequence of command hook dicts from configuration and one or more filters (before name,
-    after name, calling hook name, or a sequence of action names), filter down the command hooks to
+    after name, a sequence of action names, a sequence of sub-action steps, and/or a sequence of
-    just the ones that match the given filters.
+    execution result state names), filter down the command hooks to just the ones that match the
+    given filters.
     '''
     return tuple(
         hook_config
         for hook_config in command_hooks or ()
         for config_action_names in (hook_config.get('when'),)
+        for config_step_names in (hook_config.get('steps'),)
+        for config_state_names in (hook_config.get('states'),)
         if before is None or hook_config.get('before') == before
         if after is None or hook_config.get('after') == after
         if action_names is None
         or config_action_names is None
         or set(config_action_names or ()).intersection(set(action_names))
+        if step_names is None
+        or config_step_names is None
+        or set(config_step_names or ()).intersection(set(step_names))
+        if state_names is None
+        or config_state_names is None
+        or set(config_state_names or ()).intersection(set(state_names))
     )
 
 
-def execute_hooks(command_hooks, umask, working_directory, dry_run, **context):
+def execute_hooks(command_hooks, umask, working_directory, dry_run, **context):  # noqa: PLR0912
     '''
     Given a sequence of command hook dicts from configuration, a umask to execute with (or None), a
     working directory to execute with, and whether this is a dry run, run the commands for each
@@ -118,7 +144,7 @@ def execute_hooks(command_hooks, umask, working_directory, dry_run, **context):
                 if dry_run:
                     continue
 
-                borgmatic.execute.execute_command(
+                borgmatic.execute.execute_command(  # noqa: S604
                     [command],
                     output_log_level=(
                         logging.ERROR if hook_config.get('after') == 'error' else logging.ANSWER
@@ -143,7 +169,8 @@ class Before_after_hooks:
            before_after='do_stuff',
            umask=config.get('umask'),
            dry_run=dry_run,
-           hook_name='myhook',
+           action_names=['check'],
+           step_names=['spot'],
        ):
             do()
             some()
@@ -160,23 +187,23 @@ class Before_after_hooks:
         umask,
         working_directory,
         dry_run,
-        hook_name=None,
         action_names=None,
+        step_names=None,
         **context,
     ):
         '''
         Given a sequence of command hook configuration dicts, the before/after name, a umask to run
-        commands with, a working directory to run commands with, a dry run flag, the name of the
+        commands with, a working directory to run commands with, a dry run flag, a sequence of
-        calling hook, a sequence of action names, and any context for the executed commands, save
+        action names, a sequence of sub-action step names, and any context for the executed
-        those data points for use below.
+        commands, save those data points for use below.
         '''
         self.command_hooks = command_hooks
         self.before_after = before_after
         self.umask = umask
         self.working_directory = working_directory
         self.dry_run = dry_run
-        self.hook_name = hook_name
         self.action_names = action_names
+        self.step_names = step_names
         self.context = context
 
     def __enter__(self):
@@ -188,8 +215,8 @@ class Before_after_hooks:
                 borgmatic.hooks.command.filter_hooks(
                     self.command_hooks,
                     before=self.before_after,
-                    hook_name=self.hook_name,
                     action_names=self.action_names,
+                    step_names=self.step_names,
                 ),
                 self.umask,
                 self.working_directory,
@@ -202,7 +229,7 @@ class Before_after_hooks:
 
             # Trigger the after hook manually, since raising here will prevent it from being run
             # otherwise.
-            self.__exit__(None, None, None)
+            self.__exit__(exception_type=type(error), exception=error, traceback=None)
 
             raise ValueError(f'Error running before {self.before_after} hook: {error}')
 
@@ -215,8 +242,9 @@ class Before_after_hooks:
                 borgmatic.hooks.command.filter_hooks(
                     self.command_hooks,
                     after=self.before_after,
-                    hook_name=self.hook_name,
                     action_names=self.action_names,
+                    step_names=self.step_names,
+                    state_names=['fail' if exception_type else 'finish'],
                 ),
                 self.umask,
                 self.working_directory,

borgmatic/hooks/credential/container.py

@@ -34,7 +34,8 @@ def load_credential(hook_config, config, credential_parameters):
                 config.get('working_directory', ''),
                 (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY),
                 secret_name,
-            )
+            ),
+            encoding='utf-8',
         ) as secret_file:
             return secret_file.read().rstrip(os.linesep)
     except (FileNotFoundError, OSError) as error:

borgmatic/hooks/credential/file.py

@@ -23,7 +23,8 @@ def load_credential(hook_config, config, credential_parameters):
 
     try:
         with open(
-            os.path.join(config.get('working_directory', ''), expanded_credential_path)
+            os.path.join(config.get('working_directory', ''), expanded_credential_path),
+            encoding='utf-8',
         ) as credential_file:
             return credential_file.read().rstrip(os.linesep)
     except (FileNotFoundError, OSError) as error:

borgmatic/hooks/credential/parse.py

@@ -120,5 +120,8 @@ def resolve_credential(value, config):
         raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
 
     return borgmatic.hooks.dispatch.call_hook(
-        'load_credential', config, hook_name, tuple(credential_parameters)
+        'load_credential',
+        config,
+        hook_name,
+        tuple(credential_parameters),
     )

borgmatic/hooks/credential/systemd.py

@@ -1,6 +1,9 @@
 import logging
 import os
 import re
+import shlex
+
+import borgmatic.execute
 
 logger = logging.getLogger(__name__)
 
@@ -24,18 +27,33 @@ def load_credential(hook_config, config, credential_parameters):
 
         raise ValueError(f'Cannot load invalid credential name: "{name}"')
 
-    credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
-
-    if not credentials_directory:
-        raise ValueError(
-            f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
-        )
-
     if not CREDENTIAL_NAME_PATTERN.match(credential_name):
         raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
 
+    credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
+
+    if not credentials_directory:
+        logger.debug(
+            f'Falling back to loading credential "{credential_name}" via systemd-creds because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
+        )
+
+        command = (
+            *shlex.split((hook_config or {}).get('systemd_creds_command', 'systemd-creds')),
+            'decrypt',
+            os.path.join(
+                (hook_config or {}).get(
+                    'encrypted_credentials_directory', '/etc/credstore.encrypted'
+                ),
+                credential_name,
+            ),
+        )
+
+        return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
+
     try:
-        with open(os.path.join(credentials_directory, credential_name)) as credential_file:
+        with open(
+            os.path.join(credentials_directory, credential_name), encoding='utf-8'
+        ) as credential_file:
             return credential_file.read().rstrip(os.linesep)
     except (FileNotFoundError, OSError) as error:
         logger.warning(error)

borgmatic/hooks/data_source/bootstrap.py

@@ -1,3 +1,4 @@
+import contextlib
 import glob
 import importlib
 import json
@@ -38,7 +39,9 @@ def dump_data_sources(
         return []
 
     borgmatic_manifest_path = os.path.join(
-        borgmatic_runtime_directory, 'bootstrap', 'manifest.json'
+        borgmatic_runtime_directory,
+        'bootstrap',
+        'manifest.json',
     )
 
     if dry_run:
@@ -46,7 +49,7 @@ def dump_data_sources(
 
     os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
 
-    with open(borgmatic_manifest_path, 'w') as manifest_file:
+    with open(borgmatic_manifest_path, 'w', encoding='utf-8') as manifest_file:
         json.dump(
             {
                 'borgmatic_version': importlib.metadata.version('borgmatic'),
@@ -57,7 +60,8 @@ def dump_data_sources(
 
     patterns.extend(
         borgmatic.borg.pattern.Pattern(
-            config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK
+            config_path,
+            source=borgmatic.borg.pattern.Pattern_source.HOOK,
         )
         for config_path in config_paths
     )
@@ -65,7 +69,7 @@ def dump_data_sources(
         borgmatic.borg.pattern.Pattern(
             os.path.join(borgmatic_runtime_directory, 'bootstrap'),
             source=borgmatic.borg.pattern.Pattern_source.HOOK,
-        )
+        ),
     )
 
     return []
@@ -86,7 +90,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         'bootstrap',
     )
     logger.debug(
-        f'Looking for bootstrap manifest files to remove in {manifest_glob}{dry_run_label}'
+        f'Looking for bootstrap manifest files to remove in {manifest_glob}{dry_run_label}',
     )
 
     for manifest_directory in glob.glob(manifest_glob):
@@ -96,19 +100,18 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         if dry_run:
             continue
 
-        try:
+        with contextlib.suppress(FileNotFoundError):
             os.remove(manifest_file_path)
-        except FileNotFoundError:
-            pass
 
-        try:
+        with contextlib.suppress(FileNotFoundError):
             os.rmdir(manifest_directory)
-        except FileNotFoundError:
-            pass
 
 
 def make_data_source_dump_patterns(
-    hook_config, config, borgmatic_runtime_directory, name=None
+    hook_config,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Restores are implemented via the separate, purpose-specific "bootstrap" action rather than the

borgmatic/hooks/data_source/btrfs.py

@@ -1,5 +1,6 @@
 import collections
 import glob
+import itertools
 import json
 import logging
 import os
@@ -21,23 +22,77 @@ def use_streaming(hook_config, config):  # pragma: no cover
     return False
 
 
-def get_subvolume_mount_points(findmnt_command):
+def get_contained_subvolume_paths(btrfs_command, subvolume_path):
     '''
-    Given a findmnt command to run, get all sorted Btrfs subvolume mount points.
+    Given the path of a Btrfs subvolume, return it in a sequence along with the paths of its
+    contained subvolumes.
+
+    If the btrfs command errors, log that error and return an empty sequence.
+    '''
+    try:
+        btrfs_output = borgmatic.execute.execute_command_and_capture_output(
+            (
+                *btrfs_command.split(' '),
+                'subvolume',
+                'list',
+                subvolume_path,
+            ),
+            close_fds=True,
+        )
+    except subprocess.CalledProcessError as error:
+        logger.debug(
+            f'Ignoring Btrfs subvolume {subvolume_path} because of error listing its subvolumes: {error}',
+        )
+
+        return ()
+
+    return (
+        subvolume_path,
+        *tuple(
+            os.path.join(subvolume_path, line.split(' ')[-1])
+            for line in btrfs_output.splitlines()
+            if line.strip()
+        ),
+    )
+
+
+FINDMNT_BTRFS_ROOT_SUBVOLUME_OPTION = 'subvolid=5'
+
+
+def get_all_subvolume_paths(btrfs_command, findmnt_command):
+    '''
+    Given btrfs and findmnt commands to run, get the sorted paths for all Btrfs subvolumes on the
+    system.
     '''
     findmnt_output = borgmatic.execute.execute_command_and_capture_output(
-        tuple(findmnt_command.split(' '))
+        (
-        + (
+            *findmnt_command.split(' '),
             '-t',  # Filesystem type.
             'btrfs',
             '--json',
             '--list',  # Request a flat list instead of a nested subvolume hierarchy.
-        )
+        ),
+        close_fds=True,
     )
 
     try:
         return tuple(
-            sorted(filesystem['target'] for filesystem in json.loads(findmnt_output)['filesystems'])
+            sorted(
+                itertools.chain.from_iterable(
+                    # If findmnt gave us a Btrfs root filesystem, list the subvolumes within it.
+                    # This is necessary because findmnt only returns a subvolume's mount point
+                    # rather than its original subvolume path (which can differ). For instance,
+                    # a subvolume might exist at /mnt/subvolume but be mounted at /home/myuser.
+                    # findmnt is still useful though because it's a global way to discover all
+                    # Btrfs subvolumes—even if we have to do some additional legwork ourselves.
+                    (
+                        get_contained_subvolume_paths(btrfs_command, filesystem['target'])
+                        if FINDMNT_BTRFS_ROOT_SUBVOLUME_OPTION in filesystem['options'].split(',')
+                        else (filesystem['target'],)
+                    )
+                    for filesystem in json.loads(findmnt_output)['filesystems']
+                ),
+            ),
         )
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {findmnt_command} JSON output: {error}')
@@ -49,9 +104,15 @@ Subvolume = collections.namedtuple('Subvolume', ('path', 'contained_patterns'),
 
 
 def get_subvolume_property(btrfs_command, subvolume_path, property_name):
+    '''
+    Given a btrfs command, a subvolume path, and a property name to lookup, return the value of the
+    corresponding property.
+
+    Raise subprocess.CalledProcessError if the btrfs command errors.
+    '''
     output = borgmatic.execute.execute_command_and_capture_output(
-        tuple(btrfs_command.split(' '))
+        (
-        + (
+            *btrfs_command.split(' '),
             'property',
             'get',
             '-t',  # Type.
@@ -59,6 +120,7 @@ def get_subvolume_property(btrfs_command, subvolume_path, property_name):
             subvolume_path,
             property_name,
         ),
+        close_fds=True,
     )
 
     try:
@@ -72,19 +134,24 @@ def get_subvolume_property(btrfs_command, subvolume_path, property_name):
     }.get(value, value)
 
 
-def omit_read_only_subvolume_mount_points(btrfs_command, subvolume_paths):
+def omit_read_only_subvolume_paths(btrfs_command, subvolume_paths):
     '''
-    Given a Btrfs command to run and a sequence of Btrfs subvolume mount points, filter them down to
+    Given a Btrfs command to run and a sequence of Btrfs subvolume paths, filter them down to just
-    just those that are read-write. The idea is that Btrfs can't actually snapshot a read-only
+    those that are read-write. The idea is that Btrfs can't actually snapshot a read-only subvolume,
-    subvolume, so we should just ignore them.
+    so we should just ignore them.
     '''
     retained_subvolume_paths = []
 
     for subvolume_path in subvolume_paths:
-        if get_subvolume_property(btrfs_command, subvolume_path, 'ro'):
+        try:
-            logger.debug(f'Ignoring Btrfs subvolume {subvolume_path} because it is read-only')
+            if get_subvolume_property(btrfs_command, subvolume_path, 'ro'):
-        else:
+                logger.debug(f'Ignoring Btrfs subvolume {subvolume_path} because it is read-only')
-            retained_subvolume_paths.append(subvolume_path)
+            else:
+                retained_subvolume_paths.append(subvolume_path)
+        except subprocess.CalledProcessError as error:  # noqa: PERF203
+            logger.debug(
+                f'Error determining read-only status of Btrfs subvolume {subvolume_path}: {error}',
+            )
 
     return tuple(retained_subvolume_paths)
 
@@ -92,32 +159,34 @@ def omit_read_only_subvolume_mount_points(btrfs_command, subvolume_paths):
 def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
     '''
     Given a Btrfs command to run and a sequence of configured patterns, find the intersection
-    between the current Btrfs filesystem and subvolume mount points and the paths of any patterns.
+    between the current Btrfs filesystem and subvolume paths and the paths of any patterns.  The
-    The idea is that these pattern paths represent the requested subvolumes to snapshot.
+    idea is that these pattern paths represent the requested subvolumes to snapshot.
 
     Only include subvolumes that contain at least one root pattern sourced from borgmatic
     configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then
     return all subvolumes instead, sorted by path.
 
-    Return the result as a sequence of matching subvolume mount points.
+    Return the result as a sequence of matching Subvolume instances.
     '''
     candidate_patterns = set(patterns or ())
     subvolumes = []
 
-    # For each subvolume mount point, match it against the given patterns to find the subvolumes to
+    # For each subvolume path, match it against the given patterns to find the subvolumes to
     # backup. Sort the subvolumes from longest to shortest mount points, so longer mount points get
     # a whack at the candidate pattern piñata before their parents do. (Patterns are consumed during
     # this process, so no two subvolumes end up with the same contained patterns.)
-    for mount_point in reversed(
+    for subvolume_path in reversed(
-        omit_read_only_subvolume_mount_points(
+        omit_read_only_subvolume_paths(
-            btrfs_command, get_subvolume_mount_points(findmnt_command)
+            btrfs_command,
-        )
+            get_all_subvolume_paths(btrfs_command, findmnt_command),
+        ),
     ):
         subvolumes.extend(
-            Subvolume(mount_point, contained_patterns)
+            Subvolume(subvolume_path, contained_patterns)
             for contained_patterns in (
                 borgmatic.hooks.data_source.snapshot.get_contained_patterns(
-                    mount_point, candidate_patterns
+                    subvolume_path,
+                    candidate_patterns,
                 ),
             )
             if patterns is None
@@ -196,7 +265,10 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
     rewritten_path = initial_caret + os.path.join(
         subvolume_path,
         f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
-        '.',  # Borg 1.4+ "slashdot" hack.
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )
@@ -218,8 +290,8 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma:
     os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        tuple(btrfs_command.split(' '))
+        (
-        + (
+            *btrfs_command.split(' '),
             'subvolume',
             'snapshot',
             '-r',  # Read-only.
@@ -227,6 +299,7 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma:
             snapshot_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -241,9 +314,9 @@ def dump_data_sources(
     '''
     Given a Btrfs configuration dict, a configuration dict, the borgmatic configuration file paths,
     the borgmatic runtime directory, the configured patterns, and whether this is a dry run,
-    auto-detect and snapshot any Btrfs subvolume mount points listed in the given patterns. Also
+    auto-detect and snapshot any Btrfs subvolume paths listed in the given patterns. Also update
-    update those patterns, replacing subvolume mount points with corresponding snapshot directories
+    those patterns, replacing subvolume paths with corresponding snapshot directories so they get
-    so they get stored in the Borg archive instead.
+    stored in the Borg archive instead.
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.
 
@@ -291,13 +364,14 @@ def delete_snapshot(btrfs_command, snapshot_path):  # pragma: no cover
     Given a Btrfs command to run and the name of a snapshot path, delete it.
     '''
     borgmatic.execute.execute_command(
-        tuple(btrfs_command.split(' '))
+        (
-        + (
+            *btrfs_command.split(' '),
             'subvolume',
             'delete',
             snapshot_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -324,8 +398,8 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         logger.debug(error)
         return
 
-    # Reversing the sorted subvolumes ensures that we remove longer mount point paths of child
+    # Reversing the sorted subvolumes ensures that we remove longer paths of child subvolumes before
-    # subvolumes before the shorter mount point paths of parent subvolumes.
+    # the shorter paths of parent subvolumes.
     for subvolume in reversed(all_subvolumes):
         subvolume_snapshots_glob = borgmatic.config.paths.replace_temporary_subdirectory_with_glob(
             os.path.normpath(make_snapshot_path(subvolume.path)),
@@ -333,7 +407,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         )
 
         logger.debug(
-            f'Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}'
+            f'Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}',
         )
 
         for snapshot_path in glob.glob(subvolume_snapshots_glob):
@@ -363,7 +437,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
 
 
 def make_data_source_dump_patterns(
-    hook_config, config, borgmatic_runtime_directory, name=None
+    hook_config,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Restores aren't implemented, because stored files can be extracted directly with "extract".

borgmatic/hooks/data_source/config.py

@@ -0,0 +1,104 @@
+import json
+import logging
+import shutil
+import subprocess
+
+from borgmatic.execute import execute_command_and_capture_output
+
+IS_A_HOOK = False
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_database_option(option, data_source, connection_params=None, restore=False):
+    '''
+    Resolves a database option from the given data source configuration dict and
+    connection parameters dict. If restore is set to True it will consider the
+    `restore_<option>` instead.
+
+    Returns the resolved option or None. Can raise a ValueError if the hostname lookup
+    results in a container IP check.
+    '''
+    # Special case `hostname` since it overlaps with `container`
+    if option == 'hostname':
+        return get_hostname_from_config(data_source, connection_params, restore)
+    if connection_params and (value := connection_params.get(option)):
+        return value
+    if restore and f'restore_{option}' in data_source:
+        return data_source[f'restore_{option}']
+
+    return data_source.get(option)
+
+
+def get_hostname_from_config(data_source, connection_params=None, restore=False):
+    '''
+    Specialisation of `resolve_database_option` to handle the extra complexity of
+    the hostname option to also handle containers.
+
+    Returns a hostname/IP or raises an ValueError if a container IP lookup fails.
+    '''
+    # connection params win, full stop
+    if connection_params:
+        if container := connection_params.get('container'):
+            return get_ip_from_container(container)
+        if hostname := connection_params.get('hostname'):
+            return hostname
+    # ... then try the restore config
+    if restore:
+        if 'restore_container' in data_source:
+            return get_ip_from_container(data_source['restore_container'])
+        if 'restore_hostname' in data_source:
+            return data_source['restore_hostname']
+    # ... and finally fall back to the normal options
+    if 'container' in data_source:
+        return get_ip_from_container(data_source['container'])
+
+    return data_source.get('hostname')
+
+
+def get_ip_from_container(container):
+    '''
+    Determine the IP for a given container name via podman and docker.
+
+    Returns an IP or raises a ValueError if the lookup fails.
+    '''
+    engines = (shutil.which(engine) for engine in ('docker', 'podman'))
+    engines = [engine for engine in engines if engine]
+
+    if not engines:
+        raise ValueError("Neither 'docker' nor 'podman' could be found on the system")
+
+    last_error = None
+    for engine in engines:
+        try:
+            output = execute_command_and_capture_output(
+                (
+                    engine,
+                    'container',
+                    'inspect',
+                    '--format={{json .NetworkSettings}}',
+                    container,
+                )
+            )
+        except subprocess.CalledProcessError as error:
+            last_error = error
+            logger.debug(f"Could not find container '{container}' with engine '{engine}'")
+            continue  # Container does not exist
+
+        try:
+            network_data = json.loads(output.strip())
+        except json.JSONDecodeError as e:
+            raise ValueError(f'Could not decode JSON output from {engine}') from e
+        if main_ip := network_data.get('IPAddress'):
+            return main_ip
+        # No main IP found, try the networks
+        for network in network_data.get('Networks', {}).values():
+            if ip := network.get('IPAddress'):
+                return ip
+
+    if last_error:
+        raise last_error
+
+    raise ValueError(
+        f"Could not determine ip address for container '{container}'; running in host mode or userspace networking?"
+    )

borgmatic/hooks/data_source/dump.py

@@ -1,8 +1,11 @@
 import fnmatch
+import json
 import logging
 import os
 import shutil
 
+import borgmatic.actions.restore
+
 logger = logging.getLogger(__name__)
 
 IS_A_HOOK = False
@@ -16,7 +19,9 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam
     return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
 
 
-def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
+def make_data_source_dump_filename(
+    dump_path, name, hostname=None, port=None, container=None, label=None
+):
     '''
     Based on the given dump directory path, data source name, hostname, and port, return a filename
     to use for the data source dump. The hostname defaults to localhost.
@@ -26,9 +31,56 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
     if os.path.sep in name:
         raise ValueError(f'Invalid data source name {name}')
 
-    return os.path.join(
+    if container:
-        dump_path, (hostname or 'localhost') + ('' if port is None else f':{port}'), name
+        hostname = container
-    )
+    elif not hostname:
+        hostname = 'localhost'
+
+    identifier = label if label else hostname + ('' if port is None else f':{port}')
+
+    return os.path.join(dump_path, identifier, name)
+
+
+def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata):
+    '''
+    Given the borgmatic runtime directory, a data source hook name, and a sequence of
+    borgmatic.actions.restore.Dump instances of dump metadata, write a metadata file describing all
+    of those dumps. This metadata is being dumped so that it's available upon restore, e.g. to
+    support the user selecting which data source(s) should be restored.
+
+    Raise ValueError if writing to the file results in an operating system error.
+    '''
+    dumps_metadata_path = os.path.join(borgmatic_runtime_directory, hook_name, 'dumps.json')
+
+    try:
+        with open(dumps_metadata_path, 'w', encoding='utf-8') as metadata_file:
+            json.dump(
+                {
+                    'dumps': [dump._asdict() for dump in dumps_metadata],
+                },
+                metadata_file,
+                sort_keys=True,
+            )
+    except OSError as error:
+        raise ValueError(f'Error writing to dumps metadata at {dumps_metadata_path}: {error}')
+
+
+def parse_data_source_dumps_metadata(dumps_json, dumps_metadata_path):
+    '''
+    Given a dumps metadata JSON string as extracted from an archive and its path within the archive,
+    parse it into a tuple of borgmatic.actions.restore.Dump instances and return them.
+
+    Raise ValueError if parsing the JSON results in a JSON decode error or the data does not have
+    the expected keys.
+    '''
+    try:
+        return tuple(
+            borgmatic.actions.restore.Dump(**dump) for dump in json.loads(dumps_json)['dumps']
+        )
+    except (json.JSONDecodeError, TypeError, KeyError) as error:
+        raise ValueError(
+            f'Cannot read archive data source dumps metadata at {dumps_metadata_path} due to invalid JSON: {error}',
+        )
 
 
 def create_parent_directory_for_dump(dump_path):
@@ -67,10 +119,11 @@ def convert_glob_patterns_to_borg_pattern(patterns):
     Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg
     regular expression archive pattern as a single string like "re:etc/.*|tmp/.*".
     '''
-    # Remove the "\Z" generated by fnmatch.translate() because we don't want the pattern to match
+    # Remove the "\z" or "\Z" generated by fnmatch.translate() because we don't want the pattern to
-    # only at the end of a path, as directory format dumps require extracting files with paths
+    # match only at the end of a path, as directory format dumps require extracting files with paths
     # longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match
     # paths like "borgmatic/*/foo_databases/test/toc.dat"
     return 're:' + '|'.join(
-        fnmatch.translate(pattern.lstrip('/')).replace('\\Z', '') for pattern in patterns
+        fnmatch.translate(pattern.lstrip('/')).replace('\\z', '').replace('\\Z', '')
+        for pattern in patterns
     )

borgmatic/hooks/data_source/lvm.py

@@ -24,7 +24,8 @@ def use_streaming(hook_config, config):  # pragma: no cover
 
 BORGMATIC_SNAPSHOT_PREFIX = 'borgmatic-'
 Logical_volume = collections.namedtuple(
-    'Logical_volume', ('name', 'device_path', 'mount_point', 'contained_patterns')
+    'Logical_volume',
+    ('name', 'device_path', 'mount_point', 'contained_patterns'),
 )
 
 
@@ -44,14 +45,15 @@ def get_logical_volumes(lsblk_command, patterns=None):
         devices_info = json.loads(
             borgmatic.execute.execute_command_and_capture_output(
                 # Use lsblk instead of lvs here because lvs can't show active mounts.
-                tuple(lsblk_command.split(' '))
+                (
-                + (
+                    *lsblk_command.split(' '),
                     '--output',
                     'name,path,mountpoint,type',
                     '--json',
                     '--list',
-                )
+                ),
-            )
+                close_fds=True,
+            ),
         )
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {lsblk_command} JSON output: {error}')
@@ -72,7 +74,8 @@ def get_logical_volumes(lsblk_command, patterns=None):
             if device['mountpoint'] and device['type'] == 'lvm'
             for contained_patterns in (
                 borgmatic.hooks.data_source.snapshot.get_contained_patterns(
-                    device['mountpoint'], candidate_patterns
+                    device['mountpoint'],
+                    candidate_patterns,
                 ),
             )
             if not patterns
@@ -97,18 +100,19 @@ def snapshot_logical_volume(
     snapshot, and a snapshot size string, create a new LVM snapshot.
     '''
     borgmatic.execute.execute_command(
-        tuple(lvcreate_command.split(' '))
+        (
-        + (
+            *lvcreate_command.split(' '),
             '--snapshot',
             ('--extents' if '%' in snapshot_size else '--size'),
             snapshot_size,
             '--permission',
-            'r',  # Read-only.
+            'rw',  # Read-write in case an ext4 filesystem has orphaned files that need recovery.
             '--name',
             snapshot_name,
             logical_volume_device,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -121,14 +125,15 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # prag
     os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        tuple(mount_command.split(' '))
+        (
-        + (
+            *mount_command.split(' '),
             '-o',
             'ro',
             snapshot_device,
             snapshot_mount_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -159,9 +164,12 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc
         # /var/spool would result in overlapping snapshot patterns and therefore colliding mount
         # attempts.
         hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
-            MOUNT_POINT_HASH_LENGTH
+            MOUNT_POINT_HASH_LENGTH,
         ),
-        '.',  # Borg 1.4+ "slashdot" hack.
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )
@@ -215,7 +223,7 @@ def dump_data_sources(
     for logical_volume in requested_logical_volumes:
         snapshot_name = f'{logical_volume.name}_{snapshot_suffix}'
         logger.debug(
-            f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}'
+            f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}',
         )
 
         if not dry_run:
@@ -227,12 +235,14 @@ def dump_data_sources(
             )
 
         # Get the device path for the snapshot we just created.
-        try:
+        if not dry_run:
-            snapshot = get_snapshots(
+            try:
-                hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name
+                snapshot = get_snapshots(
-            )[0]
+                    hook_config.get('lvs_command', 'lvs'),
-        except IndexError:
+                    snapshot_name=snapshot_name,
-            raise ValueError(f'Cannot find LVM snapshot {snapshot_name}')
+                )[0]
+            except IndexError:
+                raise ValueError(f'Cannot find LVM snapshot {snapshot_name}')
 
         # Mount the snapshot into a particular named temporary directory so that the snapshot ends
         # up in the Borg archive at the "original" logical volume mount point path.
@@ -240,25 +250,29 @@ def dump_data_sources(
             normalized_runtime_directory,
             'lvm_snapshots',
             hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
-                MOUNT_POINT_HASH_LENGTH
+                MOUNT_POINT_HASH_LENGTH,
             ),
             logical_volume.mount_point.lstrip(os.path.sep),
         )
 
         logger.debug(
-            f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}'
+            f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}',
         )
 
         if dry_run:
             continue
 
         mount_snapshot(
-            hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path
+            hook_config.get('mount_command', 'mount'),
+            snapshot.device_path,
+            snapshot_mount_path,
         )
 
         for pattern in logical_volume.contained_patterns:
             snapshot_pattern = make_borg_snapshot_pattern(
-                pattern, logical_volume, normalized_runtime_directory
+                pattern,
+                logical_volume,
+                normalized_runtime_directory,
             )
 
             # Attempt to update the pattern in place, since pattern order matters to Borg.
@@ -275,8 +289,9 @@ def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
     Given a umount command to run and the mount path of a snapshot, unmount it.
     '''
     borgmatic.execute.execute_command(
-        tuple(umount_command.split(' ')) + (snapshot_mount_path,),
+        (*umount_command.split(' '), snapshot_mount_path),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -285,12 +300,13 @@ def remove_snapshot(lvremove_command, snapshot_device_path):  # pragma: no cover
     Given an lvremove command to run and the device path of a snapshot, remove it it.
     '''
     borgmatic.execute.execute_command(
-        tuple(lvremove_command.split(' '))
+        (
-        + (
+            *lvremove_command.split(' '),
             '--force',  # Suppress an interactive "are you sure?" type prompt.
             snapshot_device_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -310,16 +326,17 @@ def get_snapshots(lvs_command, snapshot_name=None):
         snapshot_info = json.loads(
             borgmatic.execute.execute_command_and_capture_output(
                 # Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
-                tuple(lvs_command.split(' '))
+                (
-                + (
+                    *lvs_command.split(' '),
                     '--report-format',
                     'json',
                     '--options',
                     'lv_name,lv_path',
                     '--select',
                     'lv_attr =~ ^s',  # Filter to just snapshots.
-                )
+                ),
-            )
+                close_fds=True,
+            ),
         )
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {lvs_command} JSON output: {error}')
@@ -336,7 +353,7 @@ def get_snapshots(lvs_command, snapshot_name=None):
         raise ValueError(f'Invalid {lvs_command} output: Missing key "{error}"')
 
 
-def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):  # noqa: PLR0912
     '''
     Given an LVM configuration dict, a configuration dict, the borgmatic runtime directory, and
     whether this is a dry run, unmount and delete any LVM snapshots created by borgmatic. If this is
@@ -374,7 +391,8 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
 
         for logical_volume in logical_volumes:
             snapshot_mount_path = os.path.join(
-                snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep)
+                snapshots_directory,
+                logical_volume.mount_point.lstrip(os.path.sep),
             )
 
             # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
@@ -407,7 +425,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                 continue
 
         if not dry_run:
-            shutil.rmtree(snapshots_directory)
+            shutil.rmtree(snapshots_directory, ignore_errors=True)
 
     # Delete snapshots.
     lvremove_command = hook_config.get('lvremove_command', 'lvremove')
@@ -433,7 +451,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
 
 
 def make_data_source_dump_patterns(
-    hook_config, config, borgmatic_runtime_directory, name=None
+    hook_config,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Restores aren't implemented, because stored files can be extracted directly with "extract".

borgmatic/hooks/data_source/mariadb.py

@@ -12,6 +12,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -24,7 +25,7 @@ def make_dump_path(base_directory):  # pragma: no cover
     return dump.make_data_source_dump_path(base_directory, 'mariadb_databases')
 
 
-DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile('^--defaults-extra-file=(?P<filename>.*)$')
+DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile(r'^--defaults-extra-file=(?P<filename>.*)$')
 
 
 def parse_extra_options(extra_options):
@@ -71,7 +72,7 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file
         (
             (f'user={username}' if username is not None else ''),
             (f'password="{escaped_password}"' if escaped_password is not None else ''),
-        )
+        ),
     ).strip()
 
     if not values:
@@ -94,7 +95,7 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file
     include = f'!include {defaults_extra_filename}\n' if defaults_extra_filename else ''
 
     read_file_descriptor, write_file_descriptor = os.pipe()
-    os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode('utf-8'))
+    os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode())
     os.close(write_file_descriptor)
 
     # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database
@@ -113,6 +114,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
     '''
     if database['name'] != 'all':
         return (database['name'],)
+
     if dry_run:
         return ()
 
@@ -120,13 +122,20 @@ def database_names_to_dump(database, config, username, password, environment, dr
         shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb')
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options'))
+    password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     show_command = (
         mariadb_show_command
-        + make_defaults_file_options(username, password, defaults_extra_filename)
+        + (
+            make_defaults_file_options(username, password, defaults_extra_filename)
+            if password_transport == 'pipe'
+            else ()
+        )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--skip-column-names', '--batch')
@@ -169,13 +178,15 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
+        hostname=database.get('hostname'),
-        database.get('port'),
+        port=database.get('port'),
+        container=database.get('container'),
+        label=database.get('label'),
     )
 
     if os.path.exists(dump_filename):
         logger.warning(
-            f'Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
+            f'Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}',
         )
         return None
 
@@ -184,14 +195,21 @@ def execute_dump_command(
         for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump')
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('options'))
+    password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     dump_command = (
         mariadb_dump_command
-        + make_defaults_file_options(username, password, defaults_extra_filename)
+        + (
+            make_defaults_file_options(username, password, defaults_extra_filename)
+            if password_transport == 'pipe'
+            else ()
+        )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--databases',)
@@ -244,20 +262,35 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping MariaDB databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         username = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('username'), config
+            database.get('username'),
+            config,
         )
         password = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('password'), config
+            database.get('password'),
+            config,
+        )
+        environment = dict(
+            os.environ,
+            **(
+                {'MYSQL_PWD': password}
+                if password and database.get('password_transport') == 'environment'
+                else {}
+            ),
         )
-        environment = dict(os.environ)
         dump_database_names = database_names_to_dump(
-            database, config, username, password, environment, dry_run
+            database,
+            config,
+            username,
+            password,
+            environment,
+            dry_run,
         )
 
         if not dump_database_names:
@@ -267,9 +300,19 @@ def dump_data_sources(
             raise ValueError('Cannot find any MariaDB databases to dump.')
 
         if database['name'] == 'all' and database.get('format'):
-            for dump_name in dump_database_names:
+            for database_name in dump_database_names:
+                dumps_metadata.append(
+                    borgmatic.actions.restore.Dump(
+                        'mariadb_databases',
+                        database_name,
+                        database.get('hostname'),
+                        database.get('port'),
+                        database.get('label'),
+                        database.get('container'),
+                    )
+                )
                 renamed_database = copy.copy(database)
-                renamed_database['name'] = dump_name
+                renamed_database['name'] = database_name
                 processes.append(
                     execute_dump_command(
                         renamed_database,
@@ -277,13 +320,23 @@ def dump_data_sources(
                         username,
                         password,
                         dump_path,
-                        (dump_name,),
+                        (database_name,),
                         environment,
                         dry_run,
                         dry_run_label,
-                    )
+                    ),
                 )
         else:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'mariadb_databases',
+                    database['name'],
+                    database.get('hostname'),
+                    database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
+                )
+            )
             processes.append(
                 execute_dump_command(
                     database,
@@ -295,22 +348,28 @@ def dump_data_sources(
                     environment,
                     dry_run,
                     dry_run_label,
-                )
+                ),
             )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mariadb_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mariadb_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return [process for process in processes if process]
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -321,7 +380,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -331,12 +393,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -357,24 +423,22 @@ def restore_data_source_dump(
     subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
+    hostname = database_config.resolve_database_option(
-        'restore_hostname', data_source.get('hostname')
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
+    port = database_config.resolve_database_option(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
+        'port', data_source, connection_params, restore=True
     )
-    tls = data_source.get('restore_tls', data_source.get('tls'))
+    tls = database_config.resolve_database_option('tls', data_source, restore=True)
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
+        database_config.resolve_database_option(
-            connection_params['username']
+            'username', data_source, connection_params, restore=True
-            or data_source.get('restore_username', data_source.get('username'))
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
+        database_config.resolve_database_option(
-            connection_params['password']
+            'password', data_source, connection_params, restore=True
-            or data_source.get('restore_password', data_source.get('password'))
         ),
         config,
     )
@@ -383,18 +447,27 @@ def restore_data_source_dump(
         shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb')
     )
     extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options'))
+    password_transport = data_source.get('password_transport', 'pipe')
     restore_command = (
         mariadb_restore_command
-        + make_defaults_file_options(username, password, defaults_extra_filename)
+        + (
+            make_defaults_file_options(username, password, defaults_extra_filename)
+            if password_transport == 'pipe'
+            else ()
+        )
         + extra_options
         + ('--batch',)
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if tls is True else ())
         + (('--skip-ssl',) if tls is False else ())
     )
-    environment = dict(os.environ)
+    environment = dict(
+        os.environ,
+        **({'MYSQL_PWD': password} if password and password_transport == 'environment' else {}),
+    )
 
     logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/data_source/mongodb.py

@@ -6,6 +6,7 @@ import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
 from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -53,14 +54,28 @@ def dump_data_sources(
     logger.info(f'Dumping MongoDB databases{dry_run_label}')
 
     processes = []
+    dumps_metadata = []
 
     for database in databases:
         name = database['name']
+        dumps_metadata.append(
+            borgmatic.actions.restore.Dump(
+                'mongodb_databases',
+                name,
+                database.get('hostname'),
+                database.get('port'),
+                database.get('label'),
+                database.get('container'),
+            )
+        )
+
         dump_filename = dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            database.get('hostname'),
+            hostname=database.get('hostname'),
-            database.get('port'),
+            port=database.get('port'),
+            container=database.get('container'),
+            label=database.get('label'),
         )
         dump_format = database.get('format', 'archive')
 
@@ -74,17 +89,22 @@ def dump_data_sources(
 
         if dump_format == 'directory':
             dump.create_parent_directory_for_dump(dump_filename)
-            execute_command(command, shell=True)
+            execute_command(command, shell=True)  # noqa: S604
         else:
             dump.create_named_pipe_for_dump(dump_filename)
-            processes.append(execute_command(command, shell=True, run_to_completion=False))
+            processes.append(
+                execute_command(command, shell=True, run_to_completion=False),  # noqa: S604
+            )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mongodb_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mongodb_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return processes
@@ -102,7 +122,7 @@ def make_password_config_file(password):
     logger.debug('Writing MongoDB password to configuration file pipe')
 
     read_file_descriptor, write_file_descriptor = os.pipe()
-    os.write(write_file_descriptor, f'password: {password}'.encode('utf-8'))
+    os.write(write_file_descriptor, f'password: {password}'.encode())
     os.close(write_file_descriptor)
 
     # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database
@@ -123,18 +143,20 @@ def build_dump_command(database, config, dump_filename, dump_format):
     dump_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump')
     )
+    hostname = database_config.resolve_database_option('hostname', database)
     return (
         dump_command
         + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
-        + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+        + (('--host', shlex.quote(hostname)) if hostname else ())
         + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
         + (
             (
                 '--username',
                 shlex.quote(
                     borgmatic.hooks.credential.parse.resolve_credential(
-                        database['username'], config
+                        database['username'],
-                    )
+                        config,
+                    ),
                 ),
             )
             if 'username' in database
@@ -142,7 +164,10 @@ def build_dump_command(database, config, dump_filename, dump_format):
         )
         + (('--config', make_password_config_file(password)) if password else ())
         + (
-            ('--authenticationDatabase', shlex.quote(database['authentication_database']))
+            (
+                '--authenticationDatabase',
+                shlex.quote(database['authentication_database']),
+            )
             if 'authentication_database' in database
             else ()
         )
@@ -157,7 +182,10 @@ def build_dump_command(database, config, dump_filename, dump_format):
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -168,7 +196,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -178,12 +209,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -211,13 +246,20 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
+        hostname=data_source.get('hostname'),
+        port=data_source.get('port'),
+        container=data_source.get('container'),
+        label=data_source.get('label'),
     )
     restore_command = build_restore_command(
-        extract_process, data_source, config, dump_filename, connection_params
+        extract_process,
+        data_source,
+        config,
+        dump_filename,
+        connection_params,
     )
 
-    logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}")
+    logger.debug(f'Restoring MongoDB database {data_source["name"]}{dry_run_label}')
     if dry_run:
         return
 
@@ -235,47 +277,56 @@ def build_restore_command(extract_process, database, config, dump_filename, conn
     '''
     Return the custom mongorestore_command from a single database configuration.
     '''
-    hostname = connection_params['hostname'] or database.get(
+    hostname = database_config.resolve_database_option(
-        'restore_hostname', database.get('hostname')
+        'hostname', database, connection_params, restore=True
+    )
+    port = database_config.resolve_database_option(
+        'port', database, connection_params, restore=True
     )
-    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
+        database_config.resolve_database_option(
-            connection_params['username']
+            'username', database, connection_params, restore=True
-            or database.get('restore_username', database.get('username'))
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
+        database_config.resolve_database_option(
-            connection_params['password']
+            'password', database, connection_params, restore=True
-            or database.get('restore_password', database.get('password'))
         ),
         config,
     )
 
-    command = list(
+    command = [
         shlex.quote(part)
         for part in shlex.split(database.get('mongorestore_command') or 'mongorestore')
-    )
+    ]
+
     if extract_process:
         command.append('--archive')
     else:
         command.extend(('--dir', dump_filename))
+
     if database['name'] != 'all':
         command.extend(('--drop',))
+
     if hostname:
         command.extend(('--host', hostname))
+
     if port:
         command.extend(('--port', str(port)))
+
     if username:
         command.extend(('--username', username))
+
     if password:
         command.extend(('--config', make_password_config_file(password)))
+
     if 'authentication_database' in database:
         command.extend(('--authenticationDatabase', database['authentication_database']))
+
     if 'restore_options' in database:
         command.extend(database['restore_options'].split(' '))
+
     if database.get('schemas'):
         for schema in database['schemas']:
             command.extend(('--nsInclude', schema))

borgmatic/hooks/data_source/mysql.py

@@ -12,6 +12,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -36,6 +37,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
     '''
     if database['name'] != 'all':
         return (database['name'],)
+
     if dry_run:
         return ()
 
@@ -45,15 +47,24 @@ def database_names_to_dump(database, config, username, password, environment, dr
     extra_options, defaults_extra_filename = (
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options'))
     )
+    password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     show_command = (
         mysql_show_command
-        + borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+        + (
-            username, password, defaults_extra_filename
+            borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+                username,
+                password,
+                defaults_extra_filename,
+            )
+            if password_transport == 'pipe'
+            else ()
         )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--skip-column-names', '--batch')
@@ -93,13 +104,15 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
+        hostname=database.get('hostname'),
-        database.get('port'),
+        port=database.get('port'),
+        container=database.get('container'),
+        label=database.get('label'),
     )
 
     if os.path.exists(dump_filename):
         logger.warning(
-            f'Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}'
+            f'Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}',
         )
         return None
 
@@ -109,16 +122,25 @@ def execute_dump_command(
     extra_options, defaults_extra_filename = (
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options'))
     )
+    password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     dump_command = (
         mysql_dump_command
-        + borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+        + (
-            username, password, defaults_extra_filename
+            borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+                username,
+                password,
+                defaults_extra_filename,
+            )
+            if password_transport == 'pipe'
+            else ()
         )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
+        + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--databases',)
@@ -171,20 +193,35 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping MySQL databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         username = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('username'), config
+            database.get('username'),
+            config,
         )
         password = borgmatic.hooks.credential.parse.resolve_credential(
-            database.get('password'), config
+            database.get('password'),
+            config,
+        )
+        environment = dict(
+            os.environ,
+            **(
+                {'MYSQL_PWD': password}
+                if password and database.get('password_transport') == 'environment'
+                else {}
+            ),
         )
-        environment = dict(os.environ)
         dump_database_names = database_names_to_dump(
-            database, config, username, password, environment, dry_run
+            database,
+            config,
+            username,
+            password,
+            environment,
+            dry_run,
         )
 
         if not dump_database_names:
@@ -194,9 +231,19 @@ def dump_data_sources(
             raise ValueError('Cannot find any MySQL databases to dump.')
 
         if database['name'] == 'all' and database.get('format'):
-            for dump_name in dump_database_names:
+            for database_name in dump_database_names:
+                dumps_metadata.append(
+                    borgmatic.actions.restore.Dump(
+                        'mysql_databases',
+                        database_name,
+                        database.get('hostname'),
+                        database.get('port'),
+                        database.get('label'),
+                        database.get('container'),
+                    )
+                )
                 renamed_database = copy.copy(database)
-                renamed_database['name'] = dump_name
+                renamed_database['name'] = database_name
                 processes.append(
                     execute_dump_command(
                         renamed_database,
@@ -204,13 +251,23 @@ def dump_data_sources(
                         username,
                         password,
                         dump_path,
-                        (dump_name,),
+                        (database_name,),
                         environment,
                         dry_run,
                         dry_run_label,
-                    )
+                    ),
                 )
         else:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'mysql_databases',
+                    database['name'],
+                    database.get('hostname'),
+                    database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
+                )
+            )
             processes.append(
                 execute_dump_command(
                     database,
@@ -222,22 +279,28 @@ def dump_data_sources(
                     environment,
                     dry_run,
                     dry_run_label,
-                )
+                ),
             )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mysql_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mysql_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return [process for process in processes if process]
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -248,7 +311,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -258,12 +324,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -284,24 +354,22 @@ def restore_data_source_dump(
     subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
+    hostname = database_config.resolve_database_option(
-        'restore_hostname', data_source.get('hostname')
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
+    port = database_config.resolve_database_option(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
+        'port', data_source, connection_params, restore=True
     )
-    tls = data_source.get('restore_tls', data_source.get('tls'))
+    tls = database_config.resolve_database_option('tls', data_source, restore=True)
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
+        database_config.resolve_database_option(
-            connection_params['username']
+            'username', data_source, connection_params, restore=True
-            or data_source.get('restore_username', data_source.get('username'))
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
+        database_config.resolve_database_option(
-            connection_params['password']
+            'password', data_source, connection_params, restore=True
-            or data_source.get('restore_password', data_source.get('password'))
         ),
         config,
     )
@@ -312,20 +380,31 @@ def restore_data_source_dump(
     extra_options, defaults_extra_filename = (
         borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options'))
     )
+    password_transport = data_source.get('password_transport', 'pipe')
     restore_command = (
         mysql_restore_command
-        + borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+        + (
-            username, password, defaults_extra_filename
+            borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+                username,
+                password,
+                defaults_extra_filename,
+            )
+            if password_transport == 'pipe'
+            else ()
         )
         + extra_options
         + ('--batch',)
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if tls is True else ())
         + (('--skip-ssl',) if tls is False else ())
     )
-    environment = dict(os.environ)
+    environment = dict(
+        os.environ,
+        **({'MYSQL_PWD': password} if password and password_transport == 'environment' else {}),
+    )
 
     logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/data_source/postgresql.py

@@ -13,6 +13,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -32,30 +33,28 @@ def make_environment(database, config, restore_connection_params=None):
     '''
     environment = dict(os.environ)
 
-    try:
+    password = database_config.resolve_database_option(
-        if restore_connection_params:
+        'password', database, restore_connection_params, restore=restore_connection_params
-            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
+    )
-                (
+
-                    restore_connection_params.get('password')
+    if password:
-                    or database.get('restore_password', database['password'])
+        environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                ),
+            password,
-                config,
+            config,
-            )
+        )
-        else:
-            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                database['password'], config
-            )
-    except (AttributeError, KeyError):
-        pass
 
     if 'ssl_mode' in database:
         environment['PGSSLMODE'] = database['ssl_mode']
+
     if 'ssl_cert' in database:
         environment['PGSSLCERT'] = database['ssl_cert']
+
     if 'ssl_key' in database:
         environment['PGSSLKEY'] = database['ssl_key']
+
     if 'ssl_root_cert' in database:
         environment['PGSSLROOTCERT'] = database['ssl_root_cert']
+
     if 'ssl_crl' in database:
         environment['PGSSLCRL'] = database['ssl_crl']
 
@@ -76,18 +75,21 @@ def database_names_to_dump(database, config, environment, dry_run):
 
     if requested_name != 'all':
         return (requested_name,)
+
     if not database.get('format'):
         return ('all',)
+
     if dry_run:
         return ()
 
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql')
     )
+    hostname = database_config.resolve_database_option('hostname', database)
     list_command = (
         psql_command
         + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (
             (
@@ -143,12 +145,13 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping PostgreSQL databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
         environment = make_environment(database, config)
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
@@ -158,6 +161,16 @@ def dump_data_sources(
             raise ValueError('Cannot find any PostgreSQL databases to dump.')
 
         for database_name in dump_database_names:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'postgresql_databases',
+                    database_name,
+                    database.get('hostname'),
+                    database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
+                )
+            )
             dump_format = database.get('format', None if database_name == 'all' else 'custom')
             compression = database.get('compression')
             default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump'
@@ -168,15 +181,19 @@ def dump_data_sources(
             dump_filename = dump.make_data_source_dump_filename(
                 dump_path,
                 database_name,
-                database.get('hostname'),
+                hostname=database.get('hostname'),
-                database.get('port'),
+                port=database.get('port'),
+                container=database.get('container'),
+                label=database.get('label'),
             )
+
             if os.path.exists(dump_filename):
                 logger.warning(
-                    f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}'
+                    f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}',
                 )
                 continue
 
+            hostname = database_config.resolve_database_option('hostname', database)
             command = (
                 dump_command
                 + (
@@ -184,15 +201,16 @@ def dump_data_sources(
                     '--clean',
                     '--if-exists',
                 )
-                + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+                + (('--host', shlex.quote(hostname)) if hostname else ())
                 + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
                 + (
                     (
                         '--username',
                         shlex.quote(
                             borgmatic.hooks.credential.parse.resolve_credential(
-                                database['username'], config
+                                database['username'],
-                            )
+                                config,
+                            ),
                         ),
                     )
                     if 'username' in database
@@ -215,14 +233,14 @@ def dump_data_sources(
             )
 
             logger.debug(
-                f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}'
+                f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}',
             )
             if dry_run:
                 continue
 
             if dump_format == 'directory':
                 dump.create_parent_directory_for_dump(dump_filename)
-                execute_command(
+                execute_command(  # noqa: S604
                     command,
                     shell=True,
                     environment=environment,
@@ -230,27 +248,33 @@ def dump_data_sources(
             else:
                 dump.create_named_pipe_for_dump(dump_filename)
                 processes.append(
-                    execute_command(
+                    execute_command(  # noqa: S604
                         command,
                         shell=True,
                         environment=environment,
                         run_to_completion=False,
-                    )
+                    ),
                 )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'postgresql_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'postgresql_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return processes
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -258,12 +282,17 @@ def remove_data_source_dumps(
     actually remove anything.
     '''
     dump.remove_data_source_dumps(
-        make_dump_path(borgmatic_runtime_directory), 'PostgreSQL', dry_run
+        make_dump_path(borgmatic_runtime_directory),
+        'PostgreSQL',
+        dry_run,
     )
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -273,12 +302,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -306,16 +339,15 @@ def restore_data_source_dump(
     hostname, port, username, and password.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
+    hostname = database_config.resolve_database_option(
-        'restore_hostname', data_source.get('hostname')
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
+    port = database_config.resolve_database_option(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
+        'port', data_source, connection_params, restore=True
     )
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
+        database_config.resolve_database_option(
-            connection_params['username']
+            'username', data_source, connection_params, restore=True
-            or data_source.get('restore_username', data_source.get('username'))
         ),
         config,
     )
@@ -324,7 +356,10 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
+        hostname=hostname,
+        port=port,
+        container=data_source.get('container'),
+        label=data_source.get('label'),
     )
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql')
@@ -333,7 +368,7 @@ def restore_data_source_dump(
         psql_command
         + ('--no-password', '--no-psqlrc', '--quiet')
         + (('--host', hostname) if hostname else ())
-        + (('--port', port) if port else ())
+        + (('--port', str(port)) if port else ())
         + (('--username', username) if username else ())
         + (('--dbname', data_source['name']) if not all_databases else ())
         + (
@@ -354,7 +389,7 @@ def restore_data_source_dump(
         + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
         + (('--dbname', data_source['name']) if not all_databases else ())
         + (('--host', hostname) if hostname else ())
-        + (('--port', port) if port else ())
+        + (('--port', str(port)) if port else ())
         + (('--username', username) if username else ())
         + (('--no-owner',) if data_source.get('no_owner', False) else ())
         + (
@@ -366,7 +401,7 @@ def restore_data_source_dump(
         + tuple(
             itertools.chain.from_iterable(('--schema', schema) for schema in data_source['schemas'])
             if data_source.get('schemas')
-            else ()
+            else (),
         )
     )
 

borgmatic/hooks/data_source/snapshot.py

@@ -8,9 +8,9 @@ def get_contained_patterns(parent_directory, candidate_patterns):
     '''
     Given a parent directory and a set of candidate patterns potentially inside it, get the subset
     of contained patterns for which the parent directory is actually the parent, a grandparent, the
-    very same directory, etc. The idea is if, say, /var/log and /var/lib are candidate pattern
+    very same directory, etc. The idea is if, say, "/var/log" and "/var/lib" are candidate pattern
-    paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then
+    paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at "/var", then
-    /var is what we want to snapshot.
+    "/var" is what we want to snapshot.
 
     If a parent directory and a candidate pattern are on different devices, skip the pattern. That's
     because any snapshot of a parent directory won't actually include "contained" directories if
@@ -18,8 +18,8 @@ def get_contained_patterns(parent_directory, candidate_patterns):
 
     For this function to work, a candidate pattern path can't have any globs or other non-literal
     characters in the initial portion of the path that matches the parent directory. For instance, a
-    parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a
+    parent directory of "/var" would match a candidate pattern path of "/var/log/*/data", but not a
-    pattern path like /v*/log/*/data.
+    pattern path like "/v*/log/*/data".
 
     The one exception is that if a regular expression pattern path starts with "^", that will get
     stripped off for purposes of matching against a parent directory.

borgmatic/hooks/data_source/sqlite.py

@@ -49,32 +49,42 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping SQLite databases{dry_run_label}')
 
     for database in databases:
         database_path = database['path']
+        dumps_metadata.append(
+            borgmatic.actions.restore.Dump(
+                'sqlite_databases', database['name'], label=database.get('label')
+            )
+        )
 
         if database['name'] == 'all':
             logger.warning('The "all" database name has no meaning for SQLite databases')
+
         if not os.path.exists(database_path):
             logger.warning(
-                f'No SQLite database at {database_path}; an empty database will be created and dumped'
+                f'No SQLite database at {database_path}; an empty database will be created and dumped',
             )
 
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        dump_filename = dump.make_data_source_dump_filename(dump_path, database['name'])
+        dump_filename = dump.make_data_source_dump_filename(
+            dump_path, database['name'], label=database.get('label')
+        )
 
         if os.path.exists(dump_filename):
             logger.warning(
-                f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
+                f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}',
             )
             continue
 
         sqlite_command = tuple(
             shlex.quote(part) for part in shlex.split(database.get('sqlite_command') or 'sqlite3')
         )
-        command = sqlite_command + (
+        command = (
+            *sqlite_command,
             shlex.quote(database_path),
             '.dump',
             '>',
@@ -82,27 +92,35 @@ def dump_data_sources(
         )
 
         logger.debug(
-            f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
+            f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}',
         )
         if dry_run:
             continue
 
         dump.create_named_pipe_for_dump(dump_filename)
-        processes.append(execute_command(command, shell=True, run_to_completion=False))
+        processes.append(
+            execute_command(command, shell=True, run_to_completion=False),  # noqa: S604
+        )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'sqlite_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'sqlite_databases'),
                 source=borgmatic.borg.pattern.Pattern_source.HOOK,
-            )
+            ),
         )
 
     return processes
 
 
 def remove_data_source_dumps(
-    databases, config, borgmatic_runtime_directory, dry_run
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    dry_run,
 ):  # pragma: no cover
     '''
     Remove all database dump files for this hook regardless of the given databases. Use the
@@ -113,7 +131,10 @@ def remove_data_source_dumps(
 
 
 def make_data_source_dump_patterns(
-    databases, config, borgmatic_runtime_directory, name=None
+    databases,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory,
@@ -123,12 +144,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, hostname='*'
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            label='*',
         ),
         dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_source_directory), name, hostname='*'
+            make_dump_path(borgmatic_source_directory),
+            name,
+            label='*',
         ),
     )
 
@@ -150,7 +175,8 @@ def restore_data_source_dump(
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
     database_path = connection_params['restore_path'] or data_source.get(
-        'restore_path', data_source.get('path')
+        'restore_path',
+        data_source.get('path'),
     )
 
     logger.debug(f'Restoring SQLite database at {database_path}{dry_run_label}')
@@ -167,7 +193,7 @@ def restore_data_source_dump(
         shlex.quote(part)
         for part in shlex.split(data_source.get('sqlite_restore_command') or 'sqlite3')
     )
-    restore_command = sqlite_restore_command + (shlex.quote(database_path),)
+    restore_command = (*sqlite_restore_command, shlex.quote(database_path))
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
     execute_command_with_processes(

borgmatic/hooks/data_source/zfs.py

@@ -45,15 +45,16 @@ def get_datasets_to_backup(zfs_command, patterns):
     Return the result as a sequence of Dataset instances, sorted by mount point.
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
-        tuple(zfs_command.split(' '))
+        (
-        + (
+            *zfs_command.split(' '),
             'list',
             '-H',
             '-t',
             'filesystem',
             '-o',
             f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}',
-        )
+        ),
+        close_fds=True,
     )
 
     try:
@@ -102,7 +103,8 @@ def get_datasets_to_backup(zfs_command, patterns):
                             else ()
                         )
                         + borgmatic.hooks.data_source.snapshot.get_contained_patterns(
-                            dataset.mount_point, candidate_patterns
+                            dataset.mount_point,
+                            candidate_patterns,
                         )
                     ),
                 )
@@ -114,7 +116,7 @@ def get_datasets_to_backup(zfs_command, patterns):
                 )
             ),
             key=lambda dataset: dataset.mount_point,
-        )
+        ),
     )
 
 
@@ -123,15 +125,16 @@ def get_all_dataset_mount_points(zfs_command):
     Given a ZFS command to run, return all ZFS datasets as a sequence of sorted mount points.
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
-        tuple(zfs_command.split(' '))
+        (
-        + (
+            *zfs_command.split(' '),
             'list',
             '-H',
             '-t',
             'filesystem',
             '-o',
             'mountpoint',
-        )
+        ),
+        close_fds=True,
     )
 
     return tuple(
@@ -141,8 +144,8 @@ def get_all_dataset_mount_points(zfs_command):
                 for line in list_output.splitlines()
                 for mount_point in (line.rstrip(),)
                 if mount_point != 'none'
-            }
+            },
-        )
+        ),
     )
 
 
@@ -152,12 +155,13 @@ def snapshot_dataset(zfs_command, full_snapshot_name):  # pragma: no cover
     snapshot.
     '''
     borgmatic.execute.execute_command(
-        tuple(zfs_command.split(' '))
+        (
-        + (
+            *zfs_command.split(' '),
             'snapshot',
             full_snapshot_name,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -170,8 +174,8 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
     os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        tuple(mount_command.split(' '))
+        (
-        + (
+            *mount_command.split(' '),
             '-t',
             'zfs',
             '-o',
@@ -180,6 +184,7 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
             snapshot_mount_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -209,7 +214,10 @@ def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory):
         # For instance, without this, snapshotting a dataset at /var and another at /var/spool would
         # result in overlapping snapshot patterns and therefore colliding mount attempts.
         hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
-        '.',  # Borg 1.4+ "slashdot" hack.
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )
@@ -261,7 +269,7 @@ def dump_data_sources(
     for dataset in requested_datasets:
         full_snapshot_name = f'{dataset.name}@{snapshot_name}'
         logger.debug(
-            f'Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}'
+            f'Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}',
         )
 
         if not dry_run:
@@ -273,25 +281,29 @@ def dump_data_sources(
             normalized_runtime_directory,
             'zfs_snapshots',
             hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(
-                MOUNT_POINT_HASH_LENGTH
+                MOUNT_POINT_HASH_LENGTH,
             ),
             dataset.mount_point.lstrip(os.path.sep),
         )
 
         logger.debug(
-            f'Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}'
+            f'Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}',
         )
 
         if dry_run:
             continue
 
         mount_snapshot(
-            hook_config.get('mount_command', 'mount'), full_snapshot_name, snapshot_mount_path
+            hook_config.get('mount_command', 'mount'),
+            full_snapshot_name,
+            snapshot_mount_path,
         )
 
         for pattern in dataset.contained_patterns:
             snapshot_pattern = make_borg_snapshot_pattern(
-                pattern, dataset, normalized_runtime_directory
+                pattern,
+                dataset,
+                normalized_runtime_directory,
             )
 
             # Attempt to update the pattern in place, since pattern order matters to Borg.
@@ -308,8 +320,9 @@ def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
     Given a umount command to run and the mount path of a snapshot, unmount it.
     '''
     borgmatic.execute.execute_command(
-        tuple(umount_command.split(' ')) + (snapshot_mount_path,),
+        (*umount_command.split(' '), snapshot_mount_path),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -319,12 +332,13 @@ def destroy_snapshot(zfs_command, full_snapshot_name):  # pragma: no cover
     it.
     '''
     borgmatic.execute.execute_command(
-        tuple(zfs_command.split(' '))
+        (
-        + (
+            *tuple(zfs_command.split(' ')),
             'destroy',
             full_snapshot_name,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -334,21 +348,22 @@ def get_all_snapshots(zfs_command):
     form "dataset@snapshot".
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
-        tuple(zfs_command.split(' '))
+        (
-        + (
+            *tuple(zfs_command.split(' ')),
             'list',
             '-H',
             '-t',
             'snapshot',
             '-o',
             'name',
-        )
+        ),
+        close_fds=True,
     )
 
     return tuple(line.rstrip() for line in list_output.splitlines())
 
 
-def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):
+def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run):  # noqa: PLR0912
     '''
     Given a ZFS configuration dict, a configuration dict, the borgmatic runtime directory, and
     whether this is a dry run, unmount and destroy any ZFS snapshots created by borgmatic. If this
@@ -437,7 +452,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
 
 
 def make_data_source_dump_patterns(
-    hook_config, config, borgmatic_runtime_directory, name=None
+    hook_config,
+    config,
+    borgmatic_runtime_directory,
+    name=None,
 ):  # pragma: no cover
     '''
     Restores aren't implemented, because stored files can be extracted directly with "extract".

borgmatic/hooks/dispatch.py

@@ -86,7 +86,7 @@ def call_hooks(function_name, config, hook_type, *args, **kwargs):
     return {
         hook_name: call_hook(function_name, config, hook_name, *args, **kwargs)
         for hook_name in get_submodule_names(
-            importlib.import_module(f'borgmatic.hooks.{hook_type.value}')
+            importlib.import_module(f'borgmatic.hooks.{hook_type.value}'),
         )
         if hook_name in config or f'{hook_name}_databases' in config
     }
@@ -105,6 +105,6 @@ def call_hooks_even_if_unconfigured(function_name, config, hook_type, *args, **k
     return {
         hook_name: call_hook(function_name, config, hook_name, *args, **kwargs)
         for hook_name in get_submodule_names(
-            importlib.import_module(f'borgmatic.hooks.{hook_type.value}')
+            importlib.import_module(f'borgmatic.hooks.{hook_type.value}'),
         )
     }

borgmatic/hooks/monitoring/apprise.py

@@ -28,8 +28,10 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
 
     borgmatic.hooks.monitoring.logs.add_handler(
         borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler(
-            HANDLER_IDENTIFIER, logs_size_limit, monitoring_log_level
+            HANDLER_IDENTIFIER,
-        )
+            logs_size_limit,
+            monitoring_log_level,
+        ),
     )
 
 
@@ -39,8 +41,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     entries. If this is a dry run, then don't actually ping anything.
     '''
     try:
-        import apprise
+        import apprise  # noqa: PLC0415
-        from apprise import NotifyFormat, NotifyType
+        from apprise import NotifyFormat, NotifyType  # noqa: PLC0415
     except ImportError:  # pragma: no cover
         logger.warning('Unable to import Apprise in monitoring hook')
         return
@@ -81,13 +83,13 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
 
     body = state_config.get('body')
 
-    if state in (
+    if state in {
         borgmatic.hooks.monitoring.monitor.State.FINISH,
         borgmatic.hooks.monitoring.monitor.State.FAIL,
         borgmatic.hooks.monitoring.monitor.State.LOG,
-    ):
+    }:
         formatted_logs = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload(
-            HANDLER_IDENTIFIER
+            HANDLER_IDENTIFIER,
         )
         if formatted_logs:
             body += f'\n\n{formatted_logs}'

borgmatic/hooks/monitoring/cronhub.py

@@ -11,15 +11,19 @@ MONITOR_STATE_TO_CRONHUB = {
     monitor.State.FINISH: 'finish',
     monitor.State.FAIL: 'fail',
 }
+TIMEOUT_SECONDS = 10
 
 
 def initialize_monitor(
-    ping_url, config, config_filename, monitoring_log_level, dry_run
+    ping_url,
+    config,
+    config_filename,
+    monitoring_log_level,
+    dry_run,
 ):  # pragma: no cover
     '''
     No initialization is necessary for this monitor.
     '''
-    pass
 
 
 def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
@@ -45,7 +49,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)
         try:
-            response = requests.get(ping_url)
+            response = requests.get(
+                ping_url,
+                timeout=TIMEOUT_SECONDS,
+                headers={'User-Agent': 'borgmatic'},
+            )
             if not response.ok:
                 response.raise_for_status()
         except requests.exceptions.RequestException as error:
@@ -56,4 +64,3 @@ def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run):  #
     '''
     No destruction is necessary for this monitor.
     '''
-    pass

borgmatic/hooks/monitoring/cronitor.py

@@ -11,15 +11,19 @@ MONITOR_STATE_TO_CRONITOR = {
     monitor.State.FINISH: 'complete',
     monitor.State.FAIL: 'fail',
 }
+TIMEOUT_SECONDS = 10
 
 
 def initialize_monitor(
-    ping_url, config, config_filename, monitoring_log_level, dry_run
+    ping_url,
+    config,
+    config_filename,
+    monitoring_log_level,
+    dry_run,
 ):  # pragma: no cover
     '''
     No initialization is necessary for this monitor.
     '''
-    pass
 
 
 def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
@@ -40,7 +44,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)
         try:
-            response = requests.get(ping_url)
+            response = requests.get(
+                ping_url,
+                timeout=TIMEOUT_SECONDS,
+                headers={'User-Agent': 'borgmatic'},
+            )
             if not response.ok:
                 response.raise_for_status()
         except requests.exceptions.RequestException as error:
@@ -51,4 +59,3 @@ def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run):  #
     '''
     No destruction is necessary for this monitor.
     '''
-    pass

borgmatic/hooks/monitoring/healthchecks.py

@@ -17,6 +17,7 @@ MONITOR_STATE_TO_HEALTHCHECKS = {
 
 DEFAULT_PING_BODY_LIMIT_BYTES = 100000
 HANDLER_IDENTIFIER = 'healthchecks'
+TIMEOUT_SECONDS = 10
 
 
 def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
@@ -36,8 +37,10 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
 
     borgmatic.hooks.monitoring.logs.add_handler(
         borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler(
-            HANDLER_IDENTIFIER, ping_body_limit, monitoring_log_level
+            HANDLER_IDENTIFIER,
-        )
+            ping_body_limit,
+            monitoring_log_level,
+        ),
     )
 
 
@@ -73,9 +76,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     logger.info(f'Pinging Healthchecks {state.name.lower()}{dry_run_label}')
     logger.debug(f'Using Healthchecks ping URL {ping_url}')
 
-    if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG):
+    if state in {monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG}:
         payload = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload(
-            HANDLER_IDENTIFIER
+            HANDLER_IDENTIFIER,
         )
     else:
         payload = ''
@@ -84,7 +87,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
         logging.getLogger('urllib3').setLevel(logging.ERROR)
         try:
             response = requests.post(
-                ping_url, data=payload.encode('utf-8'), verify=hook_config.get('verify_tls', True)
+                ping_url,
+                data=payload.encode('utf-8'),
+                verify=hook_config.get('verify_tls', True),
+                timeout=TIMEOUT_SECONDS,
+                headers={'User-Agent': 'borgmatic'},
             )
             if not response.ok:
                 response.raise_for_status()

borgmatic/hooks/monitoring/logs.py

@@ -1,3 +1,4 @@
+import contextlib
 import logging
 
 IS_A_HOOK = False
@@ -88,9 +89,7 @@ def remove_handler(identifier):
     '''
     logger = logging.getLogger()
 
-    try:
+    with contextlib.suppress(ValueError):
         logger.removeHandler(get_handler(identifier))
-    except ValueError:
-        pass
 
     logger.setLevel(min(handler.level for handler in logger.handlers))

borgmatic/hooks/monitoring/loki.py

@@ -15,6 +15,7 @@ MONITOR_STATE_TO_LOKI = {
     monitor.State.FINISH: 'Finished',
     monitor.State.FAIL: 'Failed',
 }
+TIMEOUT_SECONDS = 10
 
 # Threshold at which logs get flushed to loki
 MAX_BUFFER_LINES = 100
@@ -64,12 +65,18 @@ class Loki_log_buffer:
             # Skip as there are not logs to send yet
             return
 
-        request_body = self.to_request()
         self.root['streams'][0]['values'] = []
-        request_header = {'Content-Type': 'application/json'}
 
         try:
-            result = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
+            result = requests.post(
+                self.url,
+                data=self.to_request(),
+                timeout=TIMEOUT_SECONDS,
+                headers={
+                    'Content-Type': 'application/json',
+                    'User-Agent': 'borgmatic',
+                },
+            )
             result.raise_for_status()
         except requests.RequestException:
             logger.warning('Failed to upload logs to loki')
@@ -137,9 +144,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     Add an entry to the loki logger with the current state.
     '''
     for handler in tuple(logging.getLogger().handlers):
-        if isinstance(handler, Loki_log_handler):
+        if isinstance(handler, Loki_log_handler) and state in MONITOR_STATE_TO_LOKI:
-            if state in MONITOR_STATE_TO_LOKI.keys():
+            handler.raw(f'{MONITOR_STATE_TO_LOKI[state]} backup')
-                handler.raw(f'{MONITOR_STATE_TO_LOKI[state]} backup')
 
 
 def destroy_monitor(hook_config, config, monitoring_log_level, dry_run):