Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/998

NEWS

@@ -1,3 +1,38 @@
+1.9.11
+ * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #996: Fix the "create" action to omit the repository label prefix from Borg's output when
+   databases are enabled.
+ * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure
+   than using an environment variable.
+ * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file
+   detection.
+ * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from
+   a borgmatic configuration option (e.g. "source_directories")—not from other hooks within
+   borgmatic.
+ * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical
+   volumes.
+ * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property.
+ * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
+   source directories.
+
+1.9.10
+ * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic
+   configuration files. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #987: Fix a "list" action error when the "encryption_passcommand" option is set.
+ * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer
+   "encryption_passphrase" even if it's an empty value.
+ * #988: With the "max_duration" option or the "--max-duration" flag, run the archives and
+   repository checks separately so they don't interfere with one another. Previously, borgmatic
+   refused to run checks in this situation.
+ * #989: Fix the log message code to avoid using Python 3.10+ logging features. Now borgmatic will
+   work with Python 3.9 again.
+ * Capture and delay any log records produced before logging is fully configured, so early log
+   records don't get lost.
+ * Add support for Python 3.13.
+
 1.9.9
  * #635: Log the repository path or label on every relevant log message, not just some logs.
  * #961: When the "encryption_passcommand" option is set, call the command once from borgmatic to

README.md

@@ -56,6 +56,8 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 
 ## Integrations
 
+### Data
+
 <a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
@@ -65,6 +67,11 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://btrfs.readthedocs.io/"><img src="docs/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://sourceware.org/lvm2/"><img src="docs/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://rclone.org"><img src="docs/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+
+
+### Monitoring
+
 <a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://uptime.kuma.pet/"><img src="docs/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
@@ -76,7 +83,14 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
-<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
+
+
+### Credentials
+
+<a href="https://systemd.io/"><img src="docs/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.docker.com/"><img src="docs/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://podman.io/"><img src="docs/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://keepassxc.org/"><img src="docs/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ## Getting started

borgmatic/actions/change_passphrase.py

@@ -16,7 +16,7 @@ def run_change_passphrase(
     remote_path,
 ):
     '''
-    Run the "key change-passprhase" action for the given repository.
+    Run the "key change-passphrase" action for the given repository.
     '''
     if (
         change_passphrase_arguments.repository is None

borgmatic/actions/check.py

@@ -386,13 +386,12 @@ def collect_spot_check_source_paths(
             stream_processes=stream_processes,
         )
     )
-    borg_environment = borgmatic.borg.environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
     paths_output = borgmatic.execute.execute_command_and_capture_output(
         create_flags + create_positional_arguments,
         capture_stderr=True,
-        extra_environment=borg_environment,
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/actions/create.py

@@ -36,6 +36,7 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty
         path,
         borgmatic.borg.pattern.Pattern_type(pattern_type),
         borgmatic.borg.pattern.Pattern_style(pattern_style),
+        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
     )
 
 
@@ -51,7 +52,9 @@ def collect_patterns(config):
     try:
         return (
             tuple(
-                borgmatic.borg.pattern.Pattern(source_directory)
+                borgmatic.borg.pattern.Pattern(
+                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
+                )
                 for source_directory in config.get('source_directories', ())
             )
             + tuple(
@@ -144,6 +147,7 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None):
                         pattern.type,
                         pattern.style,
                         pattern.device,
+                        pattern.source,
                     )
                     for expanded_path in expand_directory(pattern.path, working_directory)
                 )
@@ -178,6 +182,7 @@ def device_map_patterns(patterns, working_directory=None):
                 and os.path.exists(full_path)
                 else None
             ),
+            source=pattern.source,
         )
         for pattern in patterns
         for full_path in (os.path.join(working_directory or '', pattern.path),)

borgmatic/borg/borg.py

@@ -61,7 +61,7 @@ def run_arbitrary_borg(
         tuple(shlex.quote(part) for part in full_command),
         output_file=DO_NOT_CAPTURE,
         shell=True,
-        extra_environment=dict(
+        environment=dict(
             (environment.make_environment(config) or {}),
             **{
                 'BORG_REPO': repository_path,

borgmatic/borg/break_lock.py

@@ -34,10 +34,9 @@ def break_lock(
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    borg_environment = environment.make_environment(config)
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/change_passphrase.py

@@ -56,7 +56,7 @@ def change_passphrase(
         full_command,
         output_file=borgmatic.execute.DO_NOT_CAPTURE,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config_without_passphrase),
+        environment=environment.make_environment(config_without_passphrase),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/check.py

@@ -64,15 +64,11 @@ def make_check_name_flags(checks, archive_filter_flags):
 
         ('--repository-only',)
 
-    However, if both "repository" and "archives" are in checks, then omit them from the returned
-    flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
+    However, if both "repository" and "archives" are in checks, then omit the "only" flags from the
+    returned flags because Borg does both checks by default. Note that a "data" check only works
+    along with an "archives" check.
     '''
-    if 'data' in checks:
-        data_flags = ('--verify-data',)
-        checks.update({'archives'})
-    else:
-        data_flags = ()
-
+    data_flags = ('--verify-data',) if 'data' in checks else ()
     common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
 
     if {'repository', 'archives'}.issubset(checks):
@@ -142,51 +138,51 @@ def check_archives(
     except StopIteration:
         repository_check_config = {}
 
-    if check_arguments.max_duration and 'archives' in checks:
-        raise ValueError('The archives check cannot run when the --max-duration flag is used')
-    if repository_check_config.get('max_duration') and 'archives' in checks:
-        raise ValueError(
-            'The archives check cannot run when the repository check has the max_duration option set'
-        )
-
     max_duration = check_arguments.max_duration or repository_check_config.get('max_duration')
+
     umask = config.get('umask')
-
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
-
-    full_command = (
-        (local_path, 'check')
-        + (('--repair',) if check_arguments.repair else ())
-        + (('--max-duration', str(max_duration)) if max_duration else ())
-        + make_check_name_flags(checks, archive_filter_flags)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--umask', str(umask)) if umask else ())
-        + (('--log-json',) if global_arguments.log_json else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + verbosity_flags
-        + (('--progress',) if check_arguments.progress else ())
-        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_flags(repository_path, local_borg_version)
-    )
-
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    # The Borg repair option triggers an interactive prompt, which won't work when output is
-    # captured. And progress messes with the terminal directly.
-    if check_arguments.repair or check_arguments.progress:
+    if 'data' in checks:
+        checks.add('archives')
+
+    grouped_checks = (checks,)
+
+    # If max_duration is set, then archives and repository checks need to be run separately, as Borg
+    # doesn't support --max-duration along with an archives checks.
+    if max_duration and 'archives' in checks and 'repository' in checks:
+        checks.remove('repository')
+        grouped_checks = (checks, {'repository'})
+
+    for checks_subset in grouped_checks:
+        full_command = (
+            (local_path, 'check')
+            + (('--repair',) if check_arguments.repair else ())
+            + (
+                ('--max-duration', str(max_duration))
+                if max_duration and 'repository' in checks_subset
+                else ()
+            )
+            + make_check_name_flags(checks_subset, archive_filter_flags)
+            + (('--remote-path', remote_path) if remote_path else ())
+            + (('--umask', str(umask)) if umask else ())
+            + (('--log-json',) if global_arguments.log_json else ())
+            + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+            + verbosity_flags
+            + (('--progress',) if check_arguments.progress else ())
+            + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+            + flags.make_repository_flags(repository_path, local_borg_version)
+        )
+
         execute_command(
             full_command,
-            output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
-            working_directory=working_directory,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
-        )
-    else:
-        execute_command(
-            full_command,
-            extra_environment=borg_environment,
+            # The Borg repair option triggers an interactive prompt, which won't work when output is
+            # captured. And progress messes with the terminal directly.
+            output_file=(
+                DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None
+            ),
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/compact.py

@@ -49,7 +49,7 @@ def compact_segments(
     execute_command(
         full_command,
         output_log_level=logging.INFO,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/create.py

@@ -122,52 +122,63 @@ def collect_special_file_paths(
     config,
     local_path,
     working_directory,
-    borg_environment,
     borgmatic_runtime_directory,
 ):
     '''
     Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
-    a working directory, a dict of environment variables to pass to Borg, and the borgmatic runtime
-    directory, collect the paths for any special files (character devices, block devices, and named
-    pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause
-    Borg to hang if its --read-special flag is used.
+    a working directory, and the borgmatic runtime directory, collect the paths for any special
+    files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
+    during a create. These are all paths that could cause Borg to hang if its --read-special flag is
+    used.
 
     Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
-    its own special files there for database dumps. And if the borgmatic runtime directory is
-    configured to be excluded from the files Borg backs up, error, because this means Borg won't be
-    able to consume any database dumps and therefore borgmatic will hang.
+    its own special files there for database dumps and we don't want those omitted.
+
+    Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
+    plans to backup, that means the user must have excluded the runtime directory (e.g. via
+    "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
+    consume any database dumps and therefore borgmatic will hang when it tries to do so.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
-    # files including any named pipe we've created.
+    # files including any named pipe we've created. And omit "--filter" because that can break the
+    # paths output parsing below such that path lines no longer start with th expected "- ".
     paths_output = execute_command_and_capture_output(
-        tuple(argument for argument in create_command if argument != '--exclude-nodump')
+        flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
         + ('--dry-run', '--list'),
         capture_stderr=True,
         working_directory=working_directory,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
 
+    # These are all the individual files that Borg is planning to backup as determined by the Borg
+    # create dry run above.
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.split('\n')
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
-    skip_paths = {}
+
+    # These are the subset of those files that contain the borgmatic runtime directory.
+    paths_containing_runtime_directory = {}
 
     if os.path.exists(borgmatic_runtime_directory):
-        skip_paths = {
+        paths_containing_runtime_directory = {
             path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
         }
 
-        if not skip_paths and not dry_run:
+        # If no paths to backup contain the runtime directory, it must've been excluded.
+        if not paths_containing_runtime_directory and not dry_run:
             raise ValueError(
                 f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
             )
 
     return tuple(
-        path for path in paths if special_file(path, working_directory) if path not in skip_paths
+        path
+        for path in paths
+        if special_file(path, working_directory)
+        if path not in paths_containing_runtime_directory
     )
 
 
@@ -299,7 +310,6 @@ def make_base_create_command(
         logger.warning(
             'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
-        borg_environment = environment.make_environment(config)
         working_directory = borgmatic.config.paths.get_working_directory(config)
 
         logger.debug('Collecting special file paths')
@@ -309,7 +319,6 @@ def make_base_create_command(
             config,
             local_path,
             working_directory,
-            borg_environment,
             borgmatic_runtime_directory=borgmatic_runtime_directory,
         )
 
@@ -328,6 +337,7 @@ def make_base_create_command(
                         special_file_path,
                         borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
                         borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                        source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
                     )
                     for special_file_path in special_file_paths
                 ),
@@ -396,8 +406,6 @@ def create_archive(
     # the terminal directly.
     output_file = DO_NOT_CAPTURE if progress else None
 
-    borg_environment = environment.make_environment(config)
-
     create_flags += (
         (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
         + (('--stats',) if stats and not json and not dry_run else ())
@@ -414,7 +422,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -422,7 +430,7 @@ def create_archive(
         return execute_command_and_capture_output(
             create_flags + create_positional_arguments,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -432,7 +440,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/delete.py

@@ -128,7 +128,7 @@ def delete_archives(
     borgmatic.execute.execute_command(
         command,
         output_log_level=logging.ANSWER,
-        extra_environment=borgmatic.borg.environment.make_environment(config),
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/environment.py

@@ -1,6 +1,7 @@
 import os
 
 import borgmatic.borg.passcommand
+import borgmatic.hooks.credential.parse
 
 OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
@@ -9,7 +10,6 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
-    'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
 }
@@ -26,21 +26,50 @@ DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
 
 def make_environment(config):
     '''
-    Given a borgmatic configuration dict, return its options converted to a Borg environment
-    variable dict.
+    Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it
+    with a copy of the current environment variables, and return the result.
+
+    Do not reuse this environment across multiple Borg invocations, because it can include
+    references to resources like anonymous pipes for passphrases—which can only be consumed once.
+
+    Here's how native Borg precedence works for a few of the environment variables:
+
+      1. BORG_PASSPHRASE, if set, is used first.
+      2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set.
+      3. BORG_PASSPHRASE_FD is used only if neither of the above are set.
+
+    In borgmatic, we want to simulate this precedence order, but there are some additional
+    complications. First, values can come from either configuration or from environment variables
+    set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a
+    passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we
+    want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's
+    more secure than using an environment variable (BORG_PASSPHRASE).
     '''
-    environment = {}
+    environment = dict(os.environ)
 
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
 
-        if value:
+        if value is not None:
             environment[environment_variable_name] = str(value)
 
-    passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
+    if 'encryption_passphrase' in config:
+        environment.pop('BORG_PASSPHRASE', None)
+        environment.pop('BORG_PASSCOMMAND', None)
 
-    # If the passcommand produced a passphrase, send it to Borg via an anonymous pipe.
-    if passphrase:
+    if 'encryption_passcommand' in config:
+        environment.pop('BORG_PASSCOMMAND', None)
+
+    passphrase = borgmatic.hooks.credential.parse.resolve_credential(
+        config.get('encryption_passphrase'), config
+    )
+
+    if passphrase is None:
+        passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
+
+    # If there's a passphrase (from configuration, from a configured credential, or from a
+    # configured passcommand), send it to Borg via an anonymous pipe.
+    if passphrase is not None:
         read_file_descriptor, write_file_descriptor = os.pipe()
         os.write(write_file_descriptor, passphrase.encode('utf-8'))
         os.close(write_file_descriptor)

borgmatic/borg/export_key.py

@@ -67,7 +67,7 @@ def export_key(
         full_command,
         output_file=output_file,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/export_tar.py

@@ -70,7 +70,7 @@ def export_tar_archive(
         full_command,
         output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
         output_log_level=output_log_level,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/extract.py

@@ -44,7 +44,6 @@ def extract_last_archive_dry_run(
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
-    borg_environment = environment.make_environment(config)
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
         + (('--remote-path', remote_path) if remote_path else ())
@@ -59,7 +58,7 @@ def extract_last_archive_dry_run(
 
     execute_command(
         full_extract_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
@@ -144,7 +143,6 @@ def extract_archive(
         + (tuple(paths) if paths else ())
     )
 
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
     full_destination_path = (
         os.path.join(working_directory or '', destination_path) if destination_path else None
@@ -156,7 +154,7 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=full_destination_path,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -168,7 +166,7 @@ def extract_archive(
             full_command,
             output_file=subprocess.PIPE,
             run_to_completion=False,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=full_destination_path,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -178,7 +176,7 @@ def extract_archive(
     # if the restore paths don't exist in the archive.
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=full_destination_path,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

borgmatic/borg/flags.py

@@ -156,3 +156,44 @@ def warn_for_aggressive_archive_flags(json_command, json_output):
         logger.debug(f'Cannot parse JSON output from archive command: {error}')
     except (TypeError, KeyError):
         logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
+
+
+def omit_flag(arguments, flag):
+    '''
+    Given a sequence of Borg command-line arguments, return them with the given (valueless) flag
+    omitted. For instance, if the flag is "--flag" and arguments is:
+
+        ('borg', 'create', '--flag', '--other-flag')
+
+    ... then return:
+
+        ('borg', 'create', '--other-flag')
+    '''
+    return tuple(argument for argument in arguments if argument != flag)
+
+
+def omit_flag_and_value(arguments, flag):
+    '''
+    Given a sequence of Borg command-line arguments, return them with the given flag and its
+    corresponding value omitted. For instance, if the flag is "--flag" and arguments is:
+
+        ('borg', 'create', '--flag', 'value', '--other-flag')
+
+    ... or:
+
+        ('borg', 'create', '--flag=value', '--other-flag')
+
+    ... then return:
+
+        ('borg', 'create', '--other-flag')
+    '''
+    # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two',
+    # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')).
+    # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and
+    # its value.
+    return tuple(
+        argument
+        for (previous_argument, argument) in zip((None,) + arguments, arguments)
+        if flag not in (previous_argument, argument)
+        if not argument.startswith(f'{flag}=')
+    )

borgmatic/borg/info.py

@@ -102,7 +102,7 @@ def display_archives_info(
 
     json_info = execute_command_and_capture_output(
         json_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
@@ -116,7 +116,7 @@ def display_archives_info(
     execute_command(
         main_command,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

borgmatic/borg/list.py

@@ -106,8 +106,6 @@ def capture_archive_listing(
     format to use for the output, and local and remote Borg paths, capture the
     output of listing that archive and return it as a list of file paths.
     '''
-    borg_environment = environment.make_environment(config)
-
     return tuple(
         execute_command_and_capture_output(
             make_list_command(
@@ -126,7 +124,7 @@ def capture_archive_listing(
                 local_path,
                 remote_path,
             ),
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
@@ -194,7 +192,6 @@ def list_archive(
             'The --json flag on the list action is not supported when using the --archive/--find flags.'
         )
 
-    borg_environment = environment.make_environment(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     # If there are any paths to find (and there's not a single archive already selected), start by
@@ -224,7 +221,7 @@ def list_archive(
                     local_path,
                     remote_path,
                 ),
-                extra_environment=borg_environment,
+                environment=environment.make_environment(config),
                 working_directory=borgmatic.config.paths.get_working_directory(config),
                 borg_local_path=local_path,
                 borg_exit_codes=borg_exit_codes,
@@ -260,7 +257,7 @@ def list_archive(
         execute_command(
             main_command,
             output_log_level=logging.ANSWER,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/mount.py

@@ -59,7 +59,6 @@ def mount_archive(
         + (tuple(mount_arguments.paths) if mount_arguments.paths else ())
     )
 
-    borg_environment = environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
@@ -67,7 +66,7 @@ def mount_archive(
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=borg_environment,
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
@@ -76,7 +75,7 @@ def mount_archive(
 
     execute_command(
         full_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/passcommand.py

@@ -9,21 +9,14 @@ logger = logging.getLogger(__name__)
 
 
 @functools.cache
-def run_passcommand(passcommand, passphrase_configured, working_directory):
+def run_passcommand(passcommand, working_directory):
     '''
     Run the given passcommand using the given working directory and return the passphrase produced
-    by the command. But bail first if a passphrase is already configured; this mimics Borg's
-    behavior.
+    by the command.
 
     Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
     per borgmatic invocation.
     '''
-    if passcommand and passphrase_configured:
-        logger.warning(
-            'Ignoring the "encryption_passcommand" option because "encryption_passphrase" is set'
-        )
-        return None
-
     return borgmatic.execute.execute_command_and_capture_output(
         shlex.split(passcommand),
         working_directory=working_directory,
@@ -44,7 +37,4 @@ def get_passphrase_from_passcommand(config):
     if not passcommand:
         return None
 
-    passphrase = config.get('encryption_passphrase')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    return run_passcommand(passcommand, bool(passphrase), working_directory)
+    return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config))

borgmatic/borg/pattern.py

@@ -20,12 +20,31 @@ class Pattern_style(enum.Enum):
     PATH_FULL_MATCH = 'pf'
 
 
+class Pattern_source(enum.Enum):
+    '''
+    Where the pattern came from within borgmatic. This is important because certain use cases (like
+    filesystem snapshotting) only want to consider patterns that the user actually put in a
+    configuration file and not patterns from other sources.
+    '''
+
+    # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories".
+    CONFIG = 'config'
+
+    # The pattern is generated internally within borgmatic, e.g. for special file excludes.
+    INTERNAL = 'internal'
+
+    # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump
+    # directory.
+    HOOK = 'hook'
+
+
 Pattern = collections.namedtuple(
     'Pattern',
-    ('path', 'type', 'style', 'device'),
+    ('path', 'type', 'style', 'device', 'source'),
     defaults=(
         Pattern_type.ROOT,
         Pattern_style.NONE,
         None,
+        Pattern_source.HOOK,
     ),
 )

borgmatic/borg/prune.py

@@ -96,7 +96,7 @@ def prune_archives(
     execute_command(
         full_command,
         output_log_level=output_log_level,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_create.py

@@ -98,7 +98,7 @@ def create_repository(
     execute_command(
         repo_create_command,
         output_file=DO_NOT_CAPTURE,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_delete.py

@@ -88,7 +88,7 @@ def delete_repository(
             if repo_delete_arguments.force or repo_delete_arguments.cache_only
             else borgmatic.execute.DO_NOT_CAPTURE
         ),
-        extra_environment=borgmatic.borg.environment.make_environment(config),
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_info.py

@@ -50,14 +50,13 @@ def display_repository_info(
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    extra_environment = environment.make_environment(config)
     working_directory = borgmatic.config.paths.get_working_directory(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
     if repo_info_arguments.json:
         return execute_command_and_capture_output(
             full_command,
-            extra_environment=extra_environment,
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -66,7 +65,7 @@ def display_repository_info(
         execute_command(
             full_command,
             output_log_level=logging.ANSWER,
-            extra_environment=extra_environment,
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/repo_list.py

@@ -49,7 +49,7 @@ def resolve_archive_name(
 
     output = execute_command_and_capture_output(
         full_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
@@ -140,7 +140,6 @@ def list_repository(
     return JSON output).
     '''
     borgmatic.logger.add_custom_log_levels()
-    borg_environment = environment.make_environment(config)
 
     main_command = make_repo_list_command(
         repository_path,
@@ -165,7 +164,7 @@ def list_repository(
 
     json_listing = execute_command_and_capture_output(
         json_command,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
@@ -179,7 +178,7 @@ def list_repository(
     execute_command(
         main_command,
         output_log_level=logging.ANSWER,
-        extra_environment=borg_environment,
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

borgmatic/borg/transfer.py

@@ -57,7 +57,7 @@ def transfer_archives(
         full_command,
         output_log_level=logging.ANSWER,
         output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/version.py

@@ -21,7 +21,7 @@ def local_borg_version(config, local_path='borg'):
     )
     output = execute_command_and_capture_output(
         full_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/commands/arguments.py

@@ -349,12 +349,12 @@ def make_parsers():
     global_group.add_argument(
         '--log-file-format',
         type=str,
-        help='Log format string used for log messages written to the log file',
+        help='Python format string used for log messages written to the log file',
     )
     global_group.add_argument(
         '--log-json',
         action='store_true',
-        help='Write log messages and console output as one JSON object per log line instead of formatted text',
+        help='Write Borg log messages and console output as one JSON object per log line instead of formatted text',
     )
     global_group.add_argument(
         '--override',

borgmatic/commands/borgmatic.py

@@ -43,6 +43,7 @@ from borgmatic.logger import (
     DISABLED,
     Log_prefix,
     add_custom_log_levels,
+    configure_delayed_logging,
     configure_logging,
     should_do_markup,
 )
@@ -534,13 +535,16 @@ def run_actions(
 
 def load_configurations(config_filenames, overrides=None, resolve_env=True):
     '''
-    Given a sequence of configuration filenames, load and validate each configuration file. Return
-    the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
-    a sequence of paths for all loaded configuration files (including includes), and a sequence of
-    logging.LogRecord instances containing any parse errors.
+    Given a sequence of configuration filenames, a sequence of configuration file override strings
+    in the form of "option.suboption=value", and whether to resolve environment variables, load and
+    validate each configuration file. Return the results as a tuple of: dict of configuration
+    filename to corresponding parsed configuration, a sequence of paths for all loaded configuration
+    files (including includes), and a sequence of logging.LogRecord instances containing any parse
+    errors.
 
     Log records are returned here instead of being logged directly because logging isn't yet
-    initialized at this point!
+    initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this
+    approach could change.)
     '''
     # Dict mapping from config filename to corresponding parsed config dict.
     configs = collections.OrderedDict()
@@ -562,7 +566,10 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
         )
         try:
             configs[config_filename], paths, parse_logs = validate.parse_configuration(
-                config_filename, validate.schema_filename(), overrides, resolve_env
+                config_filename,
+                validate.schema_filename(),
+                overrides,
+                resolve_env,
             )
             config_paths.update(paths)
             logs.extend(parse_logs)
@@ -880,6 +887,7 @@ def exit_with_help_link():  # pragma: no cover
 
 def main(extra_summary_logs=[]):  # pragma: no cover
     configure_signals()
+    configure_delayed_logging()
 
     try:
         arguments = parse_arguments(*sys.argv[1:])
@@ -905,9 +913,12 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         print(borgmatic.commands.completion.fish.fish_completion())
         sys.exit(0)
 
+    validate = bool('validate' in arguments)
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
     configs, config_paths, parse_logs = load_configurations(
-        config_filenames, global_arguments.overrides, global_arguments.resolve_env
+        config_filenames,
+        global_arguments.overrides,
+        resolve_env=global_arguments.resolve_env and not validate,
     )
     configuration_parse_errors = (
         (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False

borgmatic/config/load.py

@@ -69,7 +69,7 @@ def include_configuration(loader, filename_node, include_directory, config_paths
         ]
 
     raise ValueError(
-        '!include value is not supported; use a single filename or a list of filenames'
+        'The value given for the !include tag is invalid; use a single filename or a list of filenames instead'
     )
 
 

borgmatic/config/schema.yaml

@@ -250,7 +250,7 @@ properties:
             repositories that were initialized with passphrase/repokey/keyfile
             encryption. Quote the value if it contains punctuation, so it parses
             correctly. And backslash any quote or backslash literals as well.
-            Defaults to not set.
+            Defaults to not set. Supports the "{credential ...}" syntax.
         example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
     checkpoint_interval:
         type: integer
@@ -632,8 +632,8 @@ properties:
                               long-running repository check into multiple
                               partial checks. Defaults to no interruption. Only
                               applies to the "repository" check, does not check
-                              the repository index, and is not compatible with a
-                              simultaneous "archives" check or "--repair" flag.
+                              the repository index and is not compatible with
+                              the "--repair" flag.
                           example: 3600
                 - required:
                     - name
@@ -989,13 +989,15 @@ properties:
                         Username with which to connect to the database. Defaults
                         to the username of the current user. You probably want
                         to specify the "postgres" superuser here when the
-                        database name is "all".
+                        database name is "all". Supports the "{credential ...}"
+                        syntax.
                     example: dbuser
                 restore_username:
                     type: string
                     description: |
                         Username with which to restore the database. Defaults to
-                        the "username" option.
+                        the "username" option. Supports the "{credential ...}"
+                        syntax.
                     example: dbuser
                 password:
                     type: string
@@ -1003,13 +1005,15 @@ properties:
                         Password with which to connect to the database. Omitting
                         a password will only work if PostgreSQL is configured to
                         trust the configured username without a password or you
-                        create a ~/.pgpass file.
+                        create a ~/.pgpass file. Supports the "{credential ...}"
+                        syntax.
                     example: trustsome1
                 restore_password:
                     type: string
                     description: |
                         Password with which to connect to the restore database.
-                        Defaults to the "password" option.
+                        Defaults to the "password" option. Supports the
+                        "{credential ...}" syntax.
                     example: trustsome1
                 no_owner:
                     type: boolean
@@ -1169,13 +1173,15 @@ properties:
                     type: string
                     description: |
                         Username with which to connect to the database. Defaults
-                        to the username of the current user.
+                        to the username of the current user. Supports the
+                        "{credential ...}" syntax.
                     example: dbuser
                 restore_username:
                     type: string
                     description: |
                         Username with which to restore the database. Defaults to
-                        the "username" option.
+                        the "username" option. Supports the "{credential ...}"
+                        syntax.
                     example: dbuser
                 password:
                     type: string
@@ -1183,6 +1189,14 @@ properties:
                         Password with which to connect to the database. Omitting
                         a password will only work if MariaDB is configured to
                         trust the configured username without a password.
+                        Supports the "{credential ...}" syntax.
+                    example: trustsome1
+                restore_password:
+                    type: string
+                    description: |
+                        Password with which to connect to the restore database.
+                        Defaults to the "password" option. Supports the
+                        "{credential ...}" syntax.
                     example: trustsome1
                 mariadb_dump_command:
                     type: string
@@ -1201,12 +1215,6 @@ properties:
                         run a specific mariadb version (e.g., one inside a
                         running container). Defaults to "mariadb".
                     example: docker exec mariadb_container mariadb           
-                restore_password:
-                    type: string
-                    description: |
-                        Password with which to connect to the restore database.
-                        Defaults to the "password" option.
-                    example: trustsome1
                 format:
                     type: string
                     enum: ['sql']
@@ -1295,13 +1303,15 @@ properties:
                     type: string
                     description: |
                         Username with which to connect to the database. Defaults
-                        to the username of the current user.
+                        to the username of the current user. Supports the
+                        "{credential ...}" syntax.
                     example: dbuser
                 restore_username:
                     type: string
                     description: |
                         Username with which to restore the database. Defaults to
-                        the "username" option.
+                        the "username" option. Supports the "{credential ...}"
+                        syntax.
                     example: dbuser
                 password:
                     type: string
@@ -1309,12 +1319,14 @@ properties:
                         Password with which to connect to the database. Omitting
                         a password will only work if MySQL is configured to
                         trust the configured username without a password.
+                        Supports the "{credential ...}" syntax.
                     example: trustsome1
                 restore_password:
                     type: string
                     description: |
                         Password with which to connect to the restore database.
-                        Defaults to the "password" option.
+                        Defaults to the "password" option. Supports the
+                        "{credential ...}" syntax.
                     example: trustsome1
                 mysql_dump_command:
                     type: string
@@ -1451,25 +1463,29 @@ properties:
                     type: string
                     description: |
                         Username with which to connect to the database. Skip it
-                        if no authentication is needed.
+                        if no authentication is needed. Supports the
+                        "{credential ...}" syntax.
                     example: dbuser
                 restore_username:
                     type: string
                     description: |
                         Username with which to restore the database. Defaults to
-                        the "username" option.
+                        the "username" option. Supports the "{credential ...}"
+                        syntax.
                     example: dbuser
                 password:
                     type: string
                     description: |
                         Password with which to connect to the database. Skip it
-                        if no authentication is needed.
+                        if no authentication is needed. Supports the
+                        "{credential ...}" syntax.
                     example: trustsome1
                 restore_password:
                     type: string
                     description: |
                         Password with which to connect to the restore database.
-                        Defaults to the "password" option.
+                        Defaults to the "password" option. Supports the
+                        "{credential ...}" syntax.
                     example: trustsome1
                 authentication_database:
                     type: string
@@ -1528,18 +1544,20 @@ properties:
             username:
                 type: string
                 description: |
-                    The username used for authentication.
+                    The username used for authentication. Supports the
+                    "{credential ...}" syntax.
                 example: testuser
             password:
                 type: string
                 description: |
-                    The password used for authentication.
+                    The password used for authentication. Supports the
+                    "{credential ...}" syntax.
                 example: fakepassword
             access_token:
                 type: string
                 description: |
                     An ntfy access token to authenticate with instead of
-                    username/password.
+                    username/password. Supports the "{credential ...}" syntax.
                 example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2
             start:
                 type: object
@@ -1634,14 +1652,16 @@ properties:
             token:
                 type: string
                 description: |
-                    Your application's API token.
+                    Your application's API token. Supports the "{credential
+                    ...}" syntax.
                 example: 7ms6TXHpTokTou2P6x4SodDeentHRa
             user:
                 type: string
                 description: |
-                    Your user/group key (or that of your target user), viewable 
-                    when logged into your dashboard: often referred to as 
+                    Your user/group key (or that of your target user), viewable
+                    when logged into your dashboard: often referred to as
                     USER_KEY in Pushover documentation and code examples.
+                    Supports the "{credential ...}" syntax.
                 example: hwRwoWsXMBWwgrSecfa9EfPey55WSN
             start:
                 type: object
@@ -1915,19 +1935,19 @@ properties:
                 type: string
                 description: |
                     The username used for authentication. Not needed if using
-                    an API key.
+                    an API key. Supports the "{credential ...}" syntax.
                 example: testuser
             password:
                 type: string
                 description: |
                     The password used for authentication. Not needed if using
-                    an API key.
+                    an API key. Supports the "{credential ...}" syntax.
                 example: fakepassword
             api_key:
                 type: string
                 description: |
-                    The API key used for authentication. Not needed if using
-                    an username/password.
+                    The API key used for authentication. Not needed if using an
+                    username/password. Supports the "{credential ...}" syntax.
                 example: fakekey
             start:
                 type: object
@@ -2207,8 +2227,8 @@ properties:
             integration_key:
                 type: string
                 description: |
-                    PagerDuty integration key used to notify PagerDuty
-                    when a backup errors.
+                    PagerDuty integration key used to notify PagerDuty when a
+                    backup errors. Supports the "{credential ...}" syntax.
                 example: a177cad45bd374409f78906a810a3074
         description: |
             Configuration for a monitoring integration with PagerDuty. Create an
@@ -2382,3 +2402,25 @@ properties:
         description: |
             Configuration for integration with Linux LVM (Logical Volume
             Manager).
+    container:
+        type: object
+        additionalProperties: false
+        properties:
+            secrets_directory:
+                type: string
+                description: |
+                    Secrets directory to use instead of "/run/secrets".
+                example: /path/to/secrets
+        description: |
+            Configuration for integration with Docker or Podman secrets.
+    keepassxc:
+        type: object
+        additionalProperties: false
+        properties:
+            keepassxc_cli_command:
+                type: string
+                description: |
+                    Command to use instead of "keepassxc-cli".
+                example: /usr/local/bin/keepassxc-cli
+        description: |
+            Configuration for integration with the KeePassXC password manager.

borgmatic/config/validate.py

@@ -88,8 +88,9 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form
-    of "option.suboption=value", return the parsed configuration as a data structure of nested dicts
-    and lists corresponding to the schema. Example return value:
+    of "option.suboption=value", and whether to resolve environment variables, return the parsed
+    configuration as a data structure of nested dicts and lists corresponding to the schema. Example
+    return value:
 
         {
             'source_directories': ['/home', '/etc'],
@@ -124,6 +125,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
         validator = jsonschema.Draft7Validator(schema)
     except AttributeError:  # pragma: no cover
         validator = jsonschema.Draft4Validator(schema)
+
     validation_errors = tuple(validator.iter_errors(config))
 
     if validation_errors:

borgmatic/execute.py

@@ -1,7 +1,6 @@
 import collections
 import enum
 import logging
-import os
 import select
 import subprocess
 import textwrap
@@ -243,6 +242,9 @@ def mask_command_secrets(full_command):
 MAX_LOGGED_COMMAND_LENGTH = 1000
 
 
+PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG = ('BORG_', 'PG', 'MARIADB_', 'MYSQL_')
+
+
 def log_command(full_command, input_file=None, output_file=None, environment=None):
     '''
     Log the given command (a sequence of command/argument strings), along with its input/output file
@@ -251,7 +253,14 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non
     logger.debug(
         textwrap.shorten(
             ' '.join(
-                tuple(f'{key}=***' for key in (environment or {}).keys())
+                tuple(
+                    f'{key}=***'
+                    for key in (environment or {}).keys()
+                    if any(
+                        key.startswith(prefix)
+                        for prefix in PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG
+                    )
+                )
                 + mask_command_secrets(full_command)
             ),
             width=MAX_LOGGED_COMMAND_LENGTH,
@@ -274,7 +283,7 @@ def execute_command(
     output_file=None,
     input_file=None,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
@@ -284,18 +293,17 @@ def execute_command(
     Execute the given command (a sequence of command/argument strings) and log its output at the
     given log level. If an open output file object is given, then write stdout to the file and only
     log stderr. If an open input file object is given, then read stdin from the file. If shell is
-    True, execute the command within a shell. If an extra environment dict is given, then use it to
-    augment the current environment, and pass the result into the command. If a working directory is
-    given, use that as the present working directory when running the command. If a Borg local path
-    is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning
-    instead of an error. But if Borg exit codes are given as a sequence of exit code configuration
-    dicts, then use that configuration to decide what's an error and what's a warning. If run to
-    completion is False, then return the process for the command without executing it to completion.
+    True, execute the command within a shell. If an environment variables dict is given, then pass
+    it into the command. If a working directory is given, use that as the present working directory
+    when running the command. If a Borg local path is given, and the command matches it (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are
+    given as a sequence of exit code configuration dicts, then use that configuration to decide
+    what's an error and what's a warning. If run to completion is False, then return the process for
+    the command without executing it to completion.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    log_command(full_command, input_file, output_file, extra_environment)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, output_file, environment)
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
@@ -308,7 +316,7 @@ def execute_command(
         env=environment,
         cwd=working_directory,
         # Necessary for the passcommand credential hook to work.
-        close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
+        close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')),
     )
     if not run_to_completion:
         return process
@@ -327,7 +335,7 @@ def execute_command_and_capture_output(
     full_command,
     capture_stderr=False,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
@@ -335,18 +343,16 @@ def execute_command_and_capture_output(
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its
     output (stdout). If capture stderr is True, then capture and return stderr in addition to
-    stdout. If shell is True, execute the command within a shell. If an extra environment dict is
-    given, then use it to augment the current environment, and pass the result into the command. If
-    a working directory is given, use that as the present working directory when running the
-    command. If a Borg local path is given, and the command matches it (regardless of arguments),
-    treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a
-    sequence of exit code configuration dicts, then use that configuration to decide what's an error
-    and what's a warning.
+    stdout. If shell is True, execute the command within a shell. If an environment variables dict
+    is given, then pass it into the command. If a working directory is given, use that as the
+    present working directory when running the command. If a Borg local path is given, and the
+    command matches it (regardless of arguments), treat exit code 1 as a warning instead of an
+    error. But if Borg exit codes are given as a sequence of exit code configuration dicts, then use
+    that configuration to decide what's an error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    log_command(full_command, environment=extra_environment)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, environment=environment)
     command = ' '.join(full_command) if shell else full_command
 
     try:
@@ -357,7 +363,7 @@ def execute_command_and_capture_output(
             env=environment,
             cwd=working_directory,
             # Necessary for the passcommand credential hook to work.
-            close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
+            close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')),
         )
     except subprocess.CalledProcessError as error:
         if (
@@ -377,7 +383,7 @@ def execute_command_with_processes(
     output_file=None,
     input_file=None,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
@@ -391,19 +397,17 @@ def execute_command_with_processes(
     If an open output file object is given, then write stdout to the file and only log stderr. But
     if output log level is None, instead suppress logging and return the captured output for (only)
     the given command. If an open input file object is given, then read stdin from the file. If
-    shell is True, execute the command within a shell. If an extra environment dict is given, then
-    use it to augment the current environment, and pass the result into the command. If a working
-    directory is given, use that as the present working directory when running the command. If a
-    Borg local path is given, then for any matching command or process (regardless of arguments),
-    treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a
-    sequence of exit code configuration dicts, then use that configuration to decide what's an error
-    and what's a warning.
+    shell is True, execute the command within a shell. If an environment variables dict is given,
+    then pass it into the command. If a working directory is given, use that as the present working
+    directory when running the command. If a Borg local path is given, then for any matching command
+    or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if
+    Borg exit codes are given as a sequence of exit code configuration dicts, then use that
+    configuration to decide what's an error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
     upstream process.
     '''
-    log_command(full_command, input_file, output_file, extra_environment)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, output_file, environment)
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
@@ -419,7 +423,7 @@ def execute_command_with_processes(
             env=environment,
             cwd=working_directory,
             # Necessary for the passcommand credential hook to work.
-            close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
+            close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')),
         )
     except (subprocess.CalledProcessError, OSError):
         # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
@@ -430,13 +434,14 @@ def execute_command_with_processes(
                 process.kill()
         raise
 
-    captured_outputs = log_outputs(
-        tuple(processes) + (command_process,),
-        (input_file, output_file),
-        output_log_level,
-        borg_local_path,
-        borg_exit_codes,
-    )
+    with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
+        captured_outputs = log_outputs(
+            tuple(processes) + (command_process,),
+            (input_file, output_file),
+            output_log_level,
+            borg_local_path,
+            borg_exit_codes,
+        )
 
     if output_log_level is None:
         return captured_outputs.get(command_process)

borgmatic/hooks/command.py

@@ -30,16 +30,18 @@ def interpolate_context(hook_description, command, context):
 
 def make_environment(current_environment, sys_module=sys):
     '''
-    Given the existing system environment as a map from environment variable name to value, return
-    (in the same form) any extra environment variables that should be used when running command
-    hooks.
+    Given the existing system environment as a map from environment variable name to value, return a
+    copy of it, augmented with any extra environment variables that should be used when running
+    command hooks.
     '''
+    environment = dict(current_environment)
+
     # Detect whether we're running within a PyInstaller bundle. If so, set or clear LD_LIBRARY_PATH
     # based on the value of LD_LIBRARY_PATH_ORIG. This prevents library version information errors.
     if getattr(sys_module, 'frozen', False) and hasattr(sys_module, '_MEIPASS'):
-        return {'LD_LIBRARY_PATH': current_environment.get('LD_LIBRARY_PATH_ORIG', '')}
+        environment['LD_LIBRARY_PATH'] = environment.get('LD_LIBRARY_PATH_ORIG', '')
 
-    return {}
+    return environment
 
 
 def execute_hook(commands, umask, config_filename, description, dry_run, **context):
@@ -85,7 +87,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
                 [command],
                 output_log_level=(logging.ERROR if description == 'on-error' else logging.WARNING),
                 shell=True,
-                extra_environment=make_environment(os.environ),
+                environment=make_environment(os.environ),
             )
     finally:
         if original_umask:

borgmatic/hooks/credential/container.py

@@ -0,0 +1,41 @@
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+SECRET_NAME_PATTERN = re.compile(r'^\w+$')
+DEFAULT_SECRETS_DIRECTORY = '/run/secrets'
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a secret name to load, read the secret from the corresponding container secrets file
+    and return it.
+
+    Raise ValueError if the credential parameters is not one element, the secret name is invalid, or
+    the secret file cannot be read.
+    '''
+    try:
+        (secret_name,) = credential_parameters
+    except ValueError:
+        raise ValueError(f'Cannot load invalid secret name: "{' '.join(credential_parameters)}"')
+
+    if not SECRET_NAME_PATTERN.match(secret_name):
+        raise ValueError(f'Cannot load invalid secret name: "{secret_name}"')
+
+    try:
+        with open(
+            os.path.join(
+                config.get('working_directory', ''),
+                (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY),
+                secret_name,
+            )
+        ) as secret_file:
+            return secret_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load secret "{secret_name}" from file: {error.filename}')

borgmatic/hooks/credential/file.py

@@ -0,0 +1,28 @@
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a credential path to load, load the credential from file and return it.
+
+    Raise ValueError if the credential parameters is not one element or the secret file cannot be
+    read.
+    '''
+    try:
+        (credential_path,) = credential_parameters
+    except ValueError:
+        raise ValueError(f'Cannot load invalid credential: "{' '.join(credential_parameters)}"')
+
+    try:
+        with open(
+            os.path.join(config.get('working_directory', ''), credential_path)
+        ) as credential_file:
+            return credential_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load credential file: {error.filename}')

borgmatic/hooks/credential/keepassxc.py

@@ -0,0 +1,40 @@
+import logging
+import os
+import shlex
+
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a KeePassXC database path and an attribute name to load, run keepassxc-cli to fetch
+    the corresponidng KeePassXC credential and return it.
+
+    Raise ValueError if keepassxc-cli can't retrieve the credential.
+    '''
+    try:
+        (database_path, attribute_name) = credential_parameters
+    except ValueError:
+        raise ValueError(
+            f'Cannot load credential with invalid KeePassXC database path and attribute name: "{' '.join(credential_parameters)}"'
+        )
+
+    if not os.path.exists(database_path):
+        raise ValueError(
+            f'Cannot load credential because KeePassXC database path does not exist: {database_path}'
+        )
+
+    return borgmatic.execute.execute_command_and_capture_output(
+        tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli')))
+        + (
+            'show',
+            '--show-protected',
+            '--attributes',
+            'Password',
+            database_path,
+            attribute_name,
+        )
+    ).rstrip(os.linesep)

borgmatic/hooks/credential/parse.py

@@ -0,0 +1,124 @@
+import functools
+import re
+import shlex
+
+import borgmatic.hooks.dispatch
+
+IS_A_HOOK = False
+
+
+class Hash_adapter:
+    '''
+    A Hash_adapter instance wraps an unhashable object and pretends it's hashable. This is intended
+    for passing to a @functools.cache-decorated function to prevent it from complaining that an
+    argument is unhashable. It should only be used for arguments that you don't want to actually
+    impact the cache hashing, because Hash_adapter doesn't actually hash the object's contents.
+
+    Example usage:
+
+        @functools.cache
+        def func(a, b):
+            print(a, b.actual_value)
+            return a
+
+        func(5, Hash_adapter({1: 2, 3: 4}))  # Calls func(), prints, and returns.
+        func(5, Hash_adapter({1: 2, 3: 4}))  # Hits the cache and just returns the value.
+        func(5, Hash_adapter({5: 6, 7: 8}))  # Also uses cache, since the Hash_adapter is ignored.
+
+    In the above function, the "b" value is one that has been wrapped with Hash_adappter, and
+    therefore "b.actual_value" is necessary to access the original value.
+    '''
+
+    def __init__(self, actual_value):
+        self.actual_value = actual_value
+
+    def __eq__(self, other):
+        return True
+
+    def __hash__(self):
+        return 0
+
+
+UNHASHABLE_TYPES = (dict, list, set)
+
+
+def cache_ignoring_unhashable_arguments(function):
+    '''
+    A function decorator that caches calls to the decorated function but ignores any unhashable
+    arguments when performing cache lookups. This is intended to be a drop-in replacement for
+    functools.cache.
+
+    Example usage:
+
+        @cache_ignoring_unhashable_arguments
+        def func(a, b):
+            print(a, b)
+            return a
+
+        func(5, {1: 2, 3: 4})  # Calls func(), prints, and returns.
+        func(5, {1: 2, 3: 4})  # Hits the cache and just returns the value.
+        func(5, {5: 6, 7: 8})  # Also uses cache, since the unhashable value (the dict) is ignored.
+    '''
+
+    @functools.cache
+    def cached_function(*args, **kwargs):
+        return function(
+            *(arg.actual_value if isinstance(arg, Hash_adapter) else arg for arg in args),
+            **{
+                key: value.actual_value if isinstance(value, Hash_adapter) else value
+                for (key, value) in kwargs.items()
+            },
+        )
+
+    @functools.wraps(function)
+    def wrapper_function(*args, **kwargs):
+        return cached_function(
+            *(Hash_adapter(arg) if isinstance(arg, UNHASHABLE_TYPES) else arg for arg in args),
+            **{
+                key: Hash_adapter(value) if isinstance(value, UNHASHABLE_TYPES) else value
+                for (key, value) in kwargs.items()
+            },
+        )
+
+    wrapper_function.cache_clear = cached_function.cache_clear
+
+    return wrapper_function
+
+
+CREDENTIAL_PATTERN = re.compile(r'\{credential( +(?P<hook_and_parameters>.*))?\}')
+
+
+@cache_ignoring_unhashable_arguments
+def resolve_credential(value, config):
+    '''
+    Given a configuration value containing a string like "{credential hookname credentialname}" and
+    a configuration dict, resolve the credential by calling the relevant hook to get the actual
+    credential value. If the given value does not actually contain a credential tag, then return it
+    unchanged.
+
+    Cache the value (ignoring the config for purposes of caching), so repeated calls to this
+    function don't need to load the credential repeatedly.
+
+    Raise ValueError if the config could not be parsed or the credential could not be loaded.
+    '''
+    if value is None:
+        return value
+
+    matcher = CREDENTIAL_PATTERN.match(value)
+
+    if not matcher:
+        return value
+
+    hook_and_parameters = matcher.group('hook_and_parameters')
+
+    if not hook_and_parameters:
+        raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
+
+    (hook_name, *credential_parameters) = shlex.split(hook_and_parameters)
+
+    if not credential_parameters:
+        raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
+
+    return borgmatic.hooks.dispatch.call_hook(
+        'load_credential', config, hook_name, tuple(credential_parameters)
+    )

borgmatic/hooks/credential/systemd.py

@@ -0,0 +1,43 @@
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$')
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a credential name to load, read the credential from the corresponding systemd
+    credential file and return it.
+
+    Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the
+    credential name is invalid, or the credential file cannot be read.
+    '''
+    try:
+        (credential_name,) = credential_parameters
+    except ValueError:
+        raise ValueError(
+            f'Cannot load invalid credential name: "{' '.join(credential_parameters)}"'
+        )
+
+    credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
+
+    if not credentials_directory:
+        raise ValueError(
+            f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
+        )
+
+    if not CREDENTIAL_NAME_PATTERN.match(credential_name):
+        raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
+
+    try:
+        with open(os.path.join(credentials_directory, credential_name)) as credential_file:
+            return credential_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}')

borgmatic/hooks/data_source/bootstrap.py

@@ -55,9 +55,17 @@ def dump_data_sources(
             manifest_file,
         )
 
-    patterns.extend(borgmatic.borg.pattern.Pattern(config_path) for config_path in config_paths)
+    patterns.extend(
+        borgmatic.borg.pattern.Pattern(
+            config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK
+        )
+        for config_path in config_paths
+    )
     patterns.append(
-        borgmatic.borg.pattern.Pattern(os.path.join(borgmatic_runtime_directory, 'bootstrap'))
+        borgmatic.borg.pattern.Pattern(
+            os.path.join(borgmatic_runtime_directory, 'bootstrap'),
+            source=borgmatic.borg.pattern.Pattern_source.HOOK,
+        )
     )
 
     return []

borgmatic/hooks/data_source/btrfs.py

@@ -54,7 +54,9 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
     between the current Btrfs filesystem and subvolume mount points and the paths of any patterns.
     The idea is that these pattern paths represent the requested subvolumes to snapshot.
 
-    If patterns is None, then return all subvolumes, sorted by path.
+    Only include subvolumes that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then
+    return all subvolumes instead, sorted by path.
 
     Return the result as a sequence of matching subvolume mount points.
     '''
@@ -73,7 +75,12 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
                     mount_point, candidate_patterns
                 ),
             )
-            if patterns is None or contained_patterns
+            if patterns is None
+            or any(
+                pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                for pattern in contained_patterns
+            )
         )
 
     return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path))
@@ -121,6 +128,7 @@ def make_snapshot_exclude_pattern(subvolume_path):  # pragma: no cover
         ),
         borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
         borgmatic.borg.pattern.Pattern_style.FNMATCH,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -153,6 +161,7 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
         pattern.type,
         pattern.style,
         pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -198,7 +207,8 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}')
 
-    # Based on the configured patterns, determine Btrfs subvolumes to backup.
+    # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those
+    # patterns that came from actual user configuration (as opposed to, say, other hooks).
     btrfs_command = hook_config.get('btrfs_command', 'btrfs')
     findmnt_command = hook_config.get('findmnt_command', 'findmnt')
     subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns)
@@ -299,9 +309,12 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                 logger.debug(error)
                 return
 
-            # Strip off the subvolume path from the end of the snapshot path and then delete the
-            # resulting directory.
-            shutil.rmtree(snapshot_path.rsplit(subvolume.path, 1)[0])
+            # Remove the snapshot parent directory if it still exists. (It might not exist if the
+            # snapshot was for "/".)
+            snapshot_parent_dir = snapshot_path.rsplit(subvolume.path, 1)[0]
+
+            if os.path.isdir(snapshot_parent_dir):
+                shutil.rmtree(snapshot_parent_dir)
 
 
 def make_data_source_dump_patterns(

borgmatic/hooks/data_source/lvm.py

@@ -1,5 +1,6 @@
 import collections
 import glob
+import hashlib
 import json
 import logging
 import os
@@ -33,7 +34,9 @@ def get_logical_volumes(lsblk_command, patterns=None):
     between the current LVM logical volume mount points and the paths of any patterns. The idea is
     that these pattern paths represent the requested logical volumes to snapshot.
 
-    If patterns is None, include all logical volume mounts points, not just those in patterns.
+    Only include logical volumes that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, include
+    all logical volume mounts points instead, not just those in patterns.
 
     Return the result as a sequence of Logical_volume instances.
     '''
@@ -72,7 +75,12 @@ def get_logical_volumes(lsblk_command, patterns=None):
                     device['mountpoint'], candidate_patterns
                 ),
             )
-            if not patterns or contained_patterns
+            if not patterns
+            or any(
+                pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                for pattern in contained_patterns
+            )
         )
     except KeyError as error:
         raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"')
@@ -124,10 +132,14 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # prag
     )
 
 
-def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
+MOUNT_POINT_HASH_LENGTH = 10
+
+
+def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory):
     '''
-    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
-    path rewritten to be in a snapshot directory based on the given runtime directory.
+    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume
+    containing it, return a new Pattern with its path rewritten to be in a snapshot directory based
+    on both the given runtime directory and the given Logical_volume's mount point.
 
     Move any initial caret in a regular expression pattern path to the beginning, so as not to break
     the regular expression.
@@ -142,6 +154,13 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
     rewritten_path = initial_caret + os.path.join(
         normalized_runtime_directory,
         'lvm_snapshots',
+        # Including this hash prevents conflicts between snapshot patterns for different logical
+        # volumes. For instance, without this, snapshotting a logical volume at /var and another at
+        # /var/spool would result in overlapping snapshot patterns and therefore colliding mount
+        # attempts.
+        hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
+            MOUNT_POINT_HASH_LENGTH
+        ),
         '.',  # Borg 1.4+ "slashdot" hack.
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
@@ -152,6 +171,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
         pattern.type,
         pattern.style,
         pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -180,7 +200,8 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'Snapshotting LVM logical volumes{dry_run_label}')
 
-    # List logical volumes to get their mount points.
+    # List logical volumes to get their mount points, but only consider those patterns that came
+    # from actual user configuration (as opposed to, say, other hooks).
     lsblk_command = hook_config.get('lsblk_command', 'lsblk')
     requested_logical_volumes = get_logical_volumes(lsblk_command, patterns)
 
@@ -218,6 +239,9 @@ def dump_data_sources(
         snapshot_mount_path = os.path.join(
             normalized_runtime_directory,
             'lvm_snapshots',
+            hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
+                MOUNT_POINT_HASH_LENGTH
+            ),
             logical_volume.mount_point.lstrip(os.path.sep),
         )
 
@@ -233,7 +257,9 @@ def dump_data_sources(
         )
 
         for pattern in logical_volume.contained_patterns:
-            snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
+            snapshot_pattern = make_borg_snapshot_pattern(
+                pattern, logical_volume, normalized_runtime_directory
+            )
 
             # Attempt to update the pattern in place, since pattern order matters to Borg.
             try:
@@ -337,6 +363,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
             os.path.normpath(borgmatic_runtime_directory),
         ),
         'lvm_snapshots',
+        '*',
     )
     logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
     umount_command = hook_config.get('umount_command', 'umount')
@@ -349,7 +376,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
             snapshot_mount_path = os.path.join(
                 snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep)
             )
-            if not os.path.isdir(snapshot_mount_path):
+
+            # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
+            # logical volume and therefore there's nothing to unmount.
+            if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
                 continue
 
             # This might fail if the directory is already mounted, but we swallow errors here since
@@ -374,7 +404,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                 return
             except subprocess.CalledProcessError as error:
                 logger.debug(error)
-                return
+                continue
 
         if not dry_run:
             shutil.rmtree(snapshots_directory)

borgmatic/hooks/data_source/mariadb.py

@@ -5,6 +5,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.parse
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
@@ -25,11 +26,11 @@ def make_dump_path(base_directory):  # pragma: no cover
 SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
 
 
-def database_names_to_dump(database, extra_environment, dry_run):
+def database_names_to_dump(database, config, environment, dry_run):
     '''
-    Given a requested database config, return the corresponding sequence of database names to dump.
-    In the case of "all", query for the names of databases on the configured host and return them,
-    excluding any system databases that will cause problems during restore.
+    Given a requested database config and a configuration dict, return the corresponding sequence of
+    database names to dump. In the case of "all", query for the names of databases on the configured
+    host and return them, excluding any system databases that will cause problems during restore.
     '''
     if database['name'] != 'all':
         return (database['name'],)
@@ -45,14 +46,19 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            (
+                '--user',
+                borgmatic.hooks.credential.parse.resolve_credential(database['username'], config),
+            )
+            if 'username' in database
+            else ()
+        )
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
     logger.debug('Querying for "all" MariaDB databases to dump')
-    show_output = execute_command_and_capture_output(
-        show_command, extra_environment=extra_environment
-    )
+    show_output = execute_command_and_capture_output(show_command, environment=environment)
 
     return tuple(
         show_name
@@ -62,7 +68,7 @@ def database_names_to_dump(database, extra_environment, dry_run):
 
 
 def execute_dump_command(
-    database, dump_path, database_names, extra_environment, dry_run, dry_run_label
+    database, config, dump_path, database_names, environment, dry_run, dry_run_label
 ):
     '''
     Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
@@ -96,7 +102,14 @@ def execute_dump_command(
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            (
+                '--user',
+                borgmatic.hooks.credential.parse.resolve_credential(database['username'], config),
+            )
+            if 'username' in database
+            else ()
+        )
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -110,7 +123,7 @@ def execute_dump_command(
 
     return execute_command(
         dump_command,
-        extra_environment=extra_environment,
+        environment=environment,
         run_to_completion=False,
     )
 
@@ -152,8 +165,19 @@ def dump_data_sources(
 
     for database in databases:
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
-        dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
+        environment = dict(
+            os.environ,
+            **(
+                {
+                    'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(
+                        database['password'], config
+                    )
+                }
+                if 'password' in database
+                else {}
+            ),
+        )
+        dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
             if dry_run:
@@ -168,9 +192,10 @@ def dump_data_sources(
                 processes.append(
                     execute_dump_command(
                         renamed_database,
+                        config,
                         dump_path,
                         (dump_name,),
-                        extra_environment,
+                        environment,
                         dry_run,
                         dry_run_label,
                     )
@@ -179,9 +204,10 @@ def dump_data_sources(
             processes.append(
                 execute_dump_command(
                     database,
+                    config,
                     dump_path,
                     dump_database_names,
-                    extra_environment,
+                    environment,
                     dry_run,
                     dry_run_label,
                 )
@@ -190,7 +216,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'mariadb_databases')
+                os.path.join(borgmatic_runtime_directory, 'mariadb_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
@@ -251,11 +278,19 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
-    username = connection_params['username'] or data_source.get(
-        'restore_username', data_source.get('username')
+    username = borgmatic.hooks.credential.parse.resolve_credential(
+        (
+            connection_params['username']
+            or data_source.get('restore_username', data_source.get('username'))
+        ),
+        config,
     )
-    password = connection_params['password'] or data_source.get(
-        'restore_password', data_source.get('password')
+    password = borgmatic.hooks.credential.parse.resolve_credential(
+        (
+            connection_params['password']
+            or data_source.get('restore_password', data_source.get('password'))
+        ),
+        config,
     )
 
     mariadb_restore_command = tuple(
@@ -274,7 +309,7 @@ def restore_data_source_dump(
         + (('--protocol', 'tcp') if hostname or port else ())
         + (('--user', username) if username else ())
     )
-    extra_environment = {'MYSQL_PWD': password} if password else None
+    environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {}))
 
     logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}")
     if dry_run:
@@ -287,5 +322,5 @@ def restore_data_source_dump(
         [extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment=extra_environment,
+        environment=environment,
     )

borgmatic/hooks/data_source/mongodb.py

@@ -4,6 +4,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.parse
 from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks.data_source import dump
 
@@ -68,7 +69,7 @@ def dump_data_sources(
         if dry_run:
             continue
 
-        command = build_dump_command(database, dump_filename, dump_format)
+        command = build_dump_command(database, config, dump_filename, dump_format)
 
         if dump_format == 'directory':
             dump.create_parent_directory_for_dump(dump_filename)
@@ -80,14 +81,15 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'mongodb_databases')
+                os.path.join(borgmatic_runtime_directory, 'mongodb_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
     return processes
 
 
-def build_dump_command(database, dump_filename, dump_format):
+def build_dump_command(database, config, dump_filename, dump_format):
     '''
     Return the mongodump command from a single database configuration.
     '''
@@ -98,8 +100,30 @@ def build_dump_command(database, dump_filename, dump_format):
         + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
         + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
         + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
-        + (('--username', shlex.quote(database['username'])) if 'username' in database else ())
-        + (('--password', shlex.quote(database['password'])) if 'password' in database else ())
+        + (
+            (
+                '--username',
+                shlex.quote(
+                    borgmatic.hooks.credential.parse.resolve_credential(
+                        database['username'], config
+                    )
+                ),
+            )
+            if 'username' in database
+            else ()
+        )
+        + (
+            (
+                '--password',
+                shlex.quote(
+                    borgmatic.hooks.credential.parse.resolve_credential(
+                        database['password'], config
+                    )
+                ),
+            )
+            if 'password' in database
+            else ()
+        )
         + (
             ('--authenticationDatabase', shlex.quote(database['authentication_database']))
             if 'authentication_database' in database
@@ -173,7 +197,7 @@ def restore_data_source_dump(
         data_source.get('hostname'),
     )
     restore_command = build_restore_command(
-        extract_process, data_source, dump_filename, connection_params
+        extract_process, data_source, config, dump_filename, connection_params
     )
 
     logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}")
@@ -190,7 +214,7 @@ def restore_data_source_dump(
     )
 
 
-def build_restore_command(extract_process, database, dump_filename, connection_params):
+def build_restore_command(extract_process, database, config, dump_filename, connection_params):
     '''
     Return the mongorestore command from a single database configuration.
     '''
@@ -198,11 +222,19 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
         'restore_hostname', database.get('hostname')
     )
     port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
-    username = connection_params['username'] or database.get(
-        'restore_username', database.get('username')
+    username = borgmatic.hooks.credential.parse.resolve_credential(
+        (
+            connection_params['username']
+            or database.get('restore_username', database.get('username'))
+        ),
+        config,
     )
-    password = connection_params['password'] or database.get(
-        'restore_password', database.get('password')
+    password = borgmatic.hooks.credential.parse.resolve_credential(
+        (
+            connection_params['password']
+            or database.get('restore_password', database.get('password'))
+        ),
+        config,
     )
 
     command = ['mongorestore']

borgmatic/hooks/data_source/mysql.py

@@ -5,6 +5,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.parse
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
@@ -25,11 +26,11 @@ def make_dump_path(base_directory):  # pragma: no cover
 SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
 
 
-def database_names_to_dump(database, extra_environment, dry_run):
+def database_names_to_dump(database, config, environment, dry_run):
     '''
-    Given a requested database config, return the corresponding sequence of database names to dump.
-    In the case of "all", query for the names of databases on the configured host and return them,
-    excluding any system databases that will cause problems during restore.
+    Given a requested database config and a configuration dict, return the corresponding sequence of
+    database names to dump. In the case of "all", query for the names of databases on the configured
+    host and return them, excluding any system databases that will cause problems during restore.
     '''
     if database['name'] != 'all':
         return (database['name'],)
@@ -45,14 +46,19 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            (
+                '--user',
+                borgmatic.hooks.credential.parse.resolve_credential(database['username'], config),
+            )
+            if 'username' in database
+            else ()
+        )
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
     logger.debug('Querying for "all" MySQL databases to dump')
-    show_output = execute_command_and_capture_output(
-        show_command, extra_environment=extra_environment
-    )
+    show_output = execute_command_and_capture_output(show_command, environment=environment)
 
     return tuple(
         show_name
@@ -62,7 +68,7 @@ def database_names_to_dump(database, extra_environment, dry_run):
 
 
 def execute_dump_command(
-    database, dump_path, database_names, extra_environment, dry_run, dry_run_label
+    database, config, dump_path, database_names, environment, dry_run, dry_run_label
 ):
     '''
     Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a
@@ -95,7 +101,14 @@ def execute_dump_command(
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            (
+                '--user',
+                borgmatic.hooks.credential.parse.resolve_credential(database['username'], config),
+            )
+            if 'username' in database
+            else ()
+        )
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -109,7 +122,7 @@ def execute_dump_command(
 
     return execute_command(
         dump_command,
-        extra_environment=extra_environment,
+        environment=environment,
         run_to_completion=False,
     )
 
@@ -151,8 +164,19 @@ def dump_data_sources(
 
     for database in databases:
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
-        dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
+        environment = dict(
+            os.environ,
+            **(
+                {
+                    'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(
+                        database['password'], config
+                    )
+                }
+                if 'password' in database
+                else {}
+            ),
+        )
+        dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
             if dry_run:
@@ -167,9 +191,10 @@ def dump_data_sources(
                 processes.append(
                     execute_dump_command(
                         renamed_database,
+                        config,
                         dump_path,
                         (dump_name,),
-                        extra_environment,
+                        environment,
                         dry_run,
                         dry_run_label,
                     )
@@ -178,9 +203,10 @@ def dump_data_sources(
             processes.append(
                 execute_dump_command(
                     database,
+                    config,
                     dump_path,
                     dump_database_names,
-                    extra_environment,
+                    environment,
                     dry_run,
                     dry_run_label,
                 )
@@ -189,7 +215,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'mysql_databases')
+                os.path.join(borgmatic_runtime_directory, 'mysql_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
@@ -250,11 +277,19 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
-    username = connection_params['username'] or data_source.get(
-        'restore_username', data_source.get('username')
+    username = borgmatic.hooks.credential.parse.resolve_credential(
+        (
+            connection_params['username']
+            or data_source.get('restore_username', data_source.get('username'))
+        ),
+        config,
     )
-    password = connection_params['password'] or data_source.get(
-        'restore_password', data_source.get('password')
+    password = borgmatic.hooks.credential.parse.resolve_credential(
+        (
+            connection_params['password']
+            or data_source.get('restore_password', data_source.get('password'))
+        ),
+        config,
     )
 
     mysql_restore_command = tuple(
@@ -273,7 +308,7 @@ def restore_data_source_dump(
         + (('--protocol', 'tcp') if hostname or port else ())
         + (('--user', username) if username else ())
     )
-    extra_environment = {'MYSQL_PWD': password} if password else None
+    environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {}))
 
     logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}")
     if dry_run:
@@ -286,5 +321,5 @@ def restore_data_source_dump(
         [extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment=extra_environment,
+        environment=environment,
     )

borgmatic/hooks/data_source/postgresql.py

@@ -7,6 +7,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.parse
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
@@ -24,46 +25,52 @@ def make_dump_path(base_directory):  # pragma: no cover
     return dump.make_data_source_dump_path(base_directory, 'postgresql_databases')
 
 
-def make_extra_environment(database, restore_connection_params=None):
+def make_environment(database, config, restore_connection_params=None):
     '''
-    Make the extra_environment dict from the given database configuration. If restore connection
-    params are given, this is for a restore operation.
+    Make an environment dict from the current environment variables and the given database
+    configuration. If restore connection params are given, this is for a restore operation.
     '''
-    extra = dict()
+    environment = dict(os.environ)
 
     try:
         if restore_connection_params:
-            extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
-                'restore_password', database['password']
+            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
+                (
+                    restore_connection_params.get('password')
+                    or database.get('restore_password', database['password'])
+                ),
+                config,
             )
         else:
-            extra['PGPASSWORD'] = database['password']
+            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
+                database['password'], config
+            )
     except (AttributeError, KeyError):
         pass
 
     if 'ssl_mode' in database:
-        extra['PGSSLMODE'] = database['ssl_mode']
+        environment['PGSSLMODE'] = database['ssl_mode']
     if 'ssl_cert' in database:
-        extra['PGSSLCERT'] = database['ssl_cert']
+        environment['PGSSLCERT'] = database['ssl_cert']
     if 'ssl_key' in database:
-        extra['PGSSLKEY'] = database['ssl_key']
+        environment['PGSSLKEY'] = database['ssl_key']
     if 'ssl_root_cert' in database:
-        extra['PGSSLROOTCERT'] = database['ssl_root_cert']
+        environment['PGSSLROOTCERT'] = database['ssl_root_cert']
     if 'ssl_crl' in database:
-        extra['PGSSLCRL'] = database['ssl_crl']
+        environment['PGSSLCRL'] = database['ssl_crl']
 
-    return extra
+    return environment
 
 
 EXCLUDED_DATABASE_NAMES = ('template0', 'template1')
 
 
-def database_names_to_dump(database, extra_environment, dry_run):
+def database_names_to_dump(database, config, environment, dry_run):
     '''
-    Given a requested database config, return the corresponding sequence of database names to dump.
-    In the case of "all" when a database format is given, query for the names of databases on the
-    configured host and return them. For "all" without a database format, just return a sequence
-    containing "all".
+    Given a requested database config and a configuration dict, return the corresponding sequence of
+    database names to dump. In the case of "all" when a database format is given, query for the
+    names of databases on the configured host and return them. For "all" without a database format,
+    just return a sequence containing "all".
     '''
     requested_name = database['name']
 
@@ -82,13 +89,18 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--username', database['username']) if 'username' in database else ())
+        + (
+            (
+                '--username',
+                borgmatic.hooks.credential.parse.resolve_credential(database['username'], config),
+            )
+            if 'username' in database
+            else ()
+        )
         + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
     )
     logger.debug('Querying for "all" PostgreSQL databases to dump')
-    list_output = execute_command_and_capture_output(
-        list_command, extra_environment=extra_environment
-    )
+    list_output = execute_command_and_capture_output(list_command, environment=environment)
 
     return tuple(
         row[0]
@@ -135,9 +147,9 @@ def dump_data_sources(
     logger.info(f'Dumping PostgreSQL databases{dry_run_label}')
 
     for database in databases:
-        extra_environment = make_extra_environment(database)
+        environment = make_environment(database, config)
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
+        dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
             if dry_run:
@@ -174,7 +186,14 @@ def dump_data_sources(
                 + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
                 + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
                 + (
-                    ('--username', shlex.quote(database['username']))
+                    (
+                        '--username',
+                        shlex.quote(
+                            borgmatic.hooks.credential.parse.resolve_credential(
+                                database['username'], config
+                            )
+                        ),
+                    )
                     if 'username' in database
                     else ()
                 )
@@ -204,7 +223,7 @@ def dump_data_sources(
                 execute_command(
                     command,
                     shell=True,
-                    extra_environment=extra_environment,
+                    environment=environment,
                 )
             else:
                 dump.create_named_pipe_for_dump(dump_filename)
@@ -212,7 +231,7 @@ def dump_data_sources(
                     execute_command(
                         command,
                         shell=True,
-                        extra_environment=extra_environment,
+                        environment=environment,
                         run_to_completion=False,
                     )
                 )
@@ -220,7 +239,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'postgresql_databases')
+                os.path.join(borgmatic_runtime_directory, 'postgresql_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
@@ -290,8 +310,12 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
-    username = connection_params['username'] or data_source.get(
-        'restore_username', data_source.get('username')
+    username = borgmatic.hooks.credential.parse.resolve_credential(
+        (
+            connection_params['username']
+            or data_source.get('restore_username', data_source.get('username'))
+        ),
+        config,
     )
 
     all_databases = bool(data_source['name'] == 'all')
@@ -344,9 +368,7 @@ def restore_data_source_dump(
         )
     )
 
-    extra_environment = make_extra_environment(
-        data_source, restore_connection_params=connection_params
-    )
+    environment = make_environment(data_source, config, restore_connection_params=connection_params)
 
     logger.debug(f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}")
     if dry_run:
@@ -359,6 +381,6 @@ def restore_data_source_dump(
         [extract_process] if extract_process else [],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout if extract_process else None,
-        extra_environment=extra_environment,
+        environment=environment,
     )
-    execute_command(analyze_command, extra_environment=extra_environment)
+    execute_command(analyze_command, environment=environment)

borgmatic/hooks/data_source/snapshot.py

@@ -11,10 +11,10 @@ def get_contained_patterns(parent_directory, candidate_patterns):
     paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then
     /var is what we want to snapshot.
 
-    For this to work, a candidate pattern path can't have any globs or other non-literal characters
-    in the initial portion of the path that matches the parent directory. For instance, a parent
-    directory of /var would match a candidate pattern path of /var/log/*/data, but not a pattern
-    path like /v*/log/*/data.
+    For this function to work, a candidate pattern path can't have any globs or other non-literal
+    characters in the initial portion of the path that matches the parent directory. For instance, a
+    parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a
+    pattern path like /v*/log/*/data.
 
     The one exception is that if a regular expression pattern path starts with "^", that will get
     stripped off for purposes of matching against a parent directory.
@@ -31,8 +31,10 @@ def get_contained_patterns(parent_directory, candidate_patterns):
         candidate
         for candidate in candidate_patterns
         for candidate_path in (pathlib.PurePath(candidate.path.lstrip('^')),)
-        if pathlib.PurePath(parent_directory) == candidate_path
-        or pathlib.PurePath(parent_directory) in candidate_path.parents
+        if (
+            pathlib.PurePath(parent_directory) == candidate_path
+            or pathlib.PurePath(parent_directory) in candidate_path.parents
+        )
     )
     candidate_patterns -= set(contained_patterns)
 

borgmatic/hooks/data_source/sqlite.py

@@ -90,7 +90,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'sqlite_databases')
+                os.path.join(borgmatic_runtime_directory, 'sqlite_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 

borgmatic/hooks/data_source/zfs.py

@@ -1,5 +1,6 @@
 import collections
 import glob
+import hashlib
 import logging
 import os
 import shutil
@@ -38,6 +39,9 @@ def get_datasets_to_backup(zfs_command, patterns):
     pattern paths represent the requested datasets to snapshot. But also include any datasets tagged
     with a borgmatic-specific user property, whether or not they appear in the patterns.
 
+    Only include datasets that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic).
+
     Return the result as a sequence of Dataset instances, sorted by mount point.
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
@@ -48,7 +52,7 @@ def get_datasets_to_backup(zfs_command, patterns):
             '-t',
             'filesystem',
             '-o',
-            f'name,mountpoint,{BORGMATIC_USER_PROPERTY}',
+            f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}',
         )
     )
 
@@ -60,7 +64,12 @@ def get_datasets_to_backup(zfs_command, patterns):
             (
                 Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ())
                 for line in list_output.splitlines()
-                for (dataset_name, mount_point, user_property_value) in (line.rstrip().split('\t'),)
+                for (dataset_name, mount_point, can_mount, user_property_value) in (
+                    line.rstrip().split('\t'),
+                )
+                # Skip datasets that are marked "canmount=off", because mounting their snapshots will
+                # result in completely empty mount points—thereby preventing us from backing them up.
+                if can_mount == 'on'
             ),
             key=lambda dataset: dataset.mount_point,
             reverse=True,
@@ -83,7 +92,12 @@ def get_datasets_to_backup(zfs_command, patterns):
                 for contained_patterns in (
                     (
                         (
-                            (borgmatic.borg.pattern.Pattern(dataset.mount_point),)
+                            (
+                                borgmatic.borg.pattern.Pattern(
+                                    dataset.mount_point,
+                                    source=borgmatic.borg.pattern.Pattern_source.HOOK,
+                                ),
+                            )
                             if dataset.auto_backup
                             else ()
                         )
@@ -92,7 +106,12 @@ def get_datasets_to_backup(zfs_command, patterns):
                         )
                     ),
                 )
-                if contained_patterns
+                if dataset.auto_backup
+                or any(
+                    pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                    and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                    for pattern in contained_patterns
+                )
             ),
             key=lambda dataset: dataset.mount_point,
         )
@@ -155,10 +174,14 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
     )
 
 
-def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
+MOUNT_POINT_HASH_LENGTH = 10
+
+
+def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory):
     '''
-    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
-    path rewritten to be in a snapshot directory based on the given runtime directory.
+    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and the Dataset containing it,
+    return a new Pattern with its path rewritten to be in a snapshot directory based on both the
+    given runtime directory and the given Dataset's mount point.
 
     Move any initial caret in a regular expression pattern path to the beginning, so as not to break
     the regular expression.
@@ -173,6 +196,10 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
     rewritten_path = initial_caret + os.path.join(
         normalized_runtime_directory,
         'zfs_snapshots',
+        # Including this hash prevents conflicts between snapshot patterns for different datasets.
+        # For instance, without this, snapshotting a dataset at /var and another at /var/spool would
+        # result in overlapping snapshot patterns and therefore colliding mount attempts.
+        hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
         '.',  # Borg 1.4+ "slashdot" hack.
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
@@ -183,6 +210,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
         pattern.type,
         pattern.style,
         pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -209,7 +237,8 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'Snapshotting ZFS datasets{dry_run_label}')
 
-    # List ZFS datasets to get their mount points.
+    # List ZFS datasets to get their mount points, but only consider those patterns that came from
+    # actual user configuration (as opposed to, say, other hooks).
     zfs_command = hook_config.get('zfs_command', 'zfs')
     requested_datasets = get_datasets_to_backup(zfs_command, patterns)
 
@@ -234,6 +263,9 @@ def dump_data_sources(
         snapshot_mount_path = os.path.join(
             normalized_runtime_directory,
             'zfs_snapshots',
+            hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(
+                MOUNT_POINT_HASH_LENGTH
+            ),
             dataset.mount_point.lstrip(os.path.sep),
         )
 
@@ -249,7 +281,9 @@ def dump_data_sources(
         )
 
         for pattern in dataset.contained_patterns:
-            snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
+            snapshot_pattern = make_borg_snapshot_pattern(
+                pattern, dataset, normalized_runtime_directory
+            )
 
             # Attempt to update the pattern in place, since pattern order matters to Borg.
             try:
@@ -334,6 +368,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
             os.path.normpath(borgmatic_runtime_directory),
         ),
         'zfs_snapshots',
+        '*',
     )
     logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
     umount_command = hook_config.get('umount_command', 'umount')
@@ -346,7 +381,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         # child datasets before the shorter mount point paths of parent datasets.
         for mount_point in reversed(dataset_mount_points):
             snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep))
-            if not os.path.isdir(snapshot_mount_path):
+
+            # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
+            # dataset and therefore there's nothing to unmount.
+            if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
                 continue
 
             # This might fail if the path is already mounted, but we swallow errors here since we'll
@@ -370,7 +408,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                     return
                 except subprocess.CalledProcessError as error:
                     logger.debug(error)
-                    return
+                    continue
 
         if not dry_run:
             shutil.rmtree(snapshots_directory)

borgmatic/hooks/monitoring/ntfy.py

@@ -2,6 +2,8 @@ import logging
 
 import requests
 
+import borgmatic.hooks.credential.parse
+
 logger = logging.getLogger(__name__)
 
 
@@ -47,9 +49,20 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
             'X-Tags': state_config.get('tags'),
         }
 
-        username = hook_config.get('username')
-        password = hook_config.get('password')
-        access_token = hook_config.get('access_token')
+        try:
+            username = borgmatic.hooks.credential.parse.resolve_credential(
+                hook_config.get('username'), config
+            )
+            password = borgmatic.hooks.credential.parse.resolve_credential(
+                hook_config.get('password'), config
+            )
+            access_token = borgmatic.hooks.credential.parse.resolve_credential(
+                hook_config.get('access_token'), config
+            )
+        except ValueError as error:
+            logger.warning(f'Ntfy credential error: {error}')
+            return
+
         auth = None
 
         if access_token is not None:

borgmatic/hooks/monitoring/pagerduty.py

@@ -5,6 +5,7 @@ import platform
 
 import requests
 
+import borgmatic.hooks.credential.parse
 from borgmatic.hooks.monitoring import monitor
 
 logger = logging.getLogger(__name__)
@@ -39,11 +40,19 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     if dry_run:
         return
 
+    try:
+        integration_key = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('integration_key'), config
+        )
+    except ValueError as error:
+        logger.warning(f'PagerDuty credential error: {error}')
+        return
+
     hostname = platform.node()
     local_timestamp = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat()
     payload = json.dumps(
         {
-            'routing_key': hook_config['integration_key'],
+            'routing_key': integration_key,
             'event_action': 'trigger',
             'payload': {
                 'summary': f'backup failed on {hostname}',

borgmatic/hooks/monitoring/pushover.py

@@ -2,6 +2,8 @@ import logging
 
 import requests
 
+import borgmatic.hooks.credential.parse
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,8 +34,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
 
     state_config = hook_config.get(state.name.lower(), {})
 
-    token = hook_config.get('token')
-    user = hook_config.get('user')
+    try:
+        token = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('token'), config
+        )
+        user = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('user'), config)
+    except ValueError as error:
+        logger.warning(f'Pushover credential error: {error}')
+        return
 
     logger.info(f'Updating Pushover{dry_run_label}')
 

borgmatic/hooks/monitoring/zabbix.py

@@ -2,6 +2,8 @@ import logging
 
 import requests
 
+import borgmatic.hooks.credential.parse
+
 logger = logging.getLogger(__name__)
 
 
@@ -34,10 +36,21 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
         },
     )
 
+    try:
+        username = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('username'), config
+        )
+        password = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('password'), config
+        )
+        api_key = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('api_key'), config
+        )
+    except ValueError as error:
+        logger.warning(f'Zabbix credential error: {error}')
+        return
+
     server = hook_config.get('server')
-    username = hook_config.get('username')
-    password = hook_config.get('password')
-    api_key = hook_config.get('api_key')
     itemid = hook_config.get('itemid')
     host = hook_config.get('host')
     key = hook_config.get('key')

borgmatic/logger.py

@@ -87,11 +87,16 @@ class Multi_stream_handler(logging.Handler):
             handler.setLevel(level)
 
 
-class Console_no_color_formatter(logging.Formatter):
-    def __init__(self, *args, **kwargs):  # pragma: no cover
-        super(Console_no_color_formatter, self).__init__(
-            '{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs
-        )
+class Log_prefix_formatter(logging.Formatter):
+    def __init__(self, fmt='{prefix}{message}', style='{', *args, **kwargs):  # pragma: no cover
+        self.prefix = None
+
+        super(Log_prefix_formatter, self).__init__(fmt=fmt, style=style, *args, **kwargs)
+
+    def format(self, record):  # pragma: no cover
+        record.prefix = f'{self.prefix}: ' if self.prefix else ''
+
+        return super(Log_prefix_formatter, self).format(record)
 
 
 class Color(enum.Enum):
@@ -105,8 +110,9 @@ class Color(enum.Enum):
 
 class Console_color_formatter(logging.Formatter):
     def __init__(self, *args, **kwargs):
+        self.prefix = None
         super(Console_color_formatter, self).__init__(
-            '{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs
+            '{prefix}{message}', style='{', *args, **kwargs
         )
 
     def format(self, record):
@@ -124,6 +130,7 @@ class Console_color_formatter(logging.Formatter):
             .get(record.levelno)
             .value
         )
+        record.prefix = f'{self.prefix}: ' if self.prefix else ''
 
         return color_text(color, super(Console_color_formatter, self).format(record))
 
@@ -188,28 +195,32 @@ def add_custom_log_levels():  # pragma: no cover
 
 def get_log_prefix():
     '''
-    Return the current log prefix from the defaults for the formatter on the first logging handler,
-    set by set_log_prefix(). Return None if no such prefix exists.
+    Return the current log prefix set by set_log_prefix(). Return None if no such prefix exists.
+
+    It would be a whole lot easier to use logger.Formatter(defaults=...) instead, but that argument
+    doesn't exist until Python 3.10+.
     '''
     try:
-        return next(
-            handler.formatter._style._defaults.get('prefix').rstrip().rstrip(':')
+        formatter = next(
+            handler.formatter
             for handler in logging.getLogger().handlers
+            if handler.formatter
+            if hasattr(handler.formatter, 'prefix')
         )
-    except (StopIteration, AttributeError):
+    except StopIteration:
         return None
 
+    return formatter.prefix
+
 
 def set_log_prefix(prefix):
     '''
-    Given a log prefix as a string, set it into the defaults for the formatters on all logging
-    handlers. Note that this overwrites any existing defaults.
+    Given a log prefix as a string, set it into the each handler's formatter so that it can inject
+    the prefix into each logged record.
     '''
     for handler in logging.getLogger().handlers:
-        try:
-            handler.formatter._style._defaults = {'prefix': f'{prefix}: ' if prefix else ''}
-        except AttributeError:
-            pass
+        if handler.formatter and hasattr(handler.formatter, 'prefix'):
+            handler.formatter.prefix = prefix
 
 
 class Log_prefix:
@@ -252,6 +263,67 @@ class Log_prefix:
         set_log_prefix(self.original_prefix)
 
 
+class Delayed_logging_handler(logging.handlers.BufferingHandler):
+    '''
+    A logging handler that buffers logs and doesn't flush them until explicitly flushed (after
+    target handlers are actually set). It's useful for holding onto messages logged before logging
+    is configured, ensuring those records eventually make their way to the relevant logging
+    handlers.
+
+    When flushing, don't forward log records to a target handler if the record's log level is below
+    that of the handler. This recreates the standard logging behavior of, say, logging.DEBUG records
+    getting suppressed if a handler's level is only set to logging.INFO.
+    '''
+
+    def __init__(self):
+        super(Delayed_logging_handler, self).__init__(capacity=0)
+
+        self.targets = None
+
+    def shouldFlush(self, record):
+        return self.targets is not None
+
+    def flush(self):
+        self.acquire()
+
+        try:
+            if not self.targets:
+                return
+
+            for record in self.buffer:
+                for target in self.targets:
+                    if record.levelno >= target.level:
+                        target.handle(record)
+
+            self.buffer.clear()
+        finally:
+            self.release()
+
+
+def configure_delayed_logging():  # pragma: no cover
+    '''
+    Configure a delayed logging handler to buffer anything that gets logged until we're ready to
+    deal with it.
+    '''
+    logging.basicConfig(
+        level=logging.DEBUG,
+        handlers=[Delayed_logging_handler()],
+    )
+
+
+def flush_delayed_logging(target_handlers):
+    '''
+    Flush any previously buffered logs to our "real" logging handlers.
+    '''
+    root_logger = logging.getLogger()
+
+    if root_logger.handlers and isinstance(root_logger.handlers[0], Delayed_logging_handler):
+        delayed_handler = root_logger.handlers[0]
+        delayed_handler.targets = target_handlers
+        delayed_handler.flush()
+        root_logger.removeHandler(delayed_handler)
+
+
 def configure_logging(
     console_log_level,
     syslog_log_level=None,
@@ -296,10 +368,9 @@ def configure_logging(
     if color_enabled:
         console_handler.setFormatter(Console_color_formatter())
     else:
-        console_handler.setFormatter(Console_no_color_formatter())
+        console_handler.setFormatter(Log_prefix_formatter())
 
     console_handler.setLevel(console_log_level)
-
     handlers = [console_handler]
 
     if syslog_log_level != logging.DISABLED:
@@ -315,10 +386,8 @@ def configure_logging(
         if syslog_path:
             syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
             syslog_handler.setFormatter(
-                logging.Formatter(
+                Log_prefix_formatter(
                     'borgmatic: {levelname} {prefix}{message}',  # noqa: FS003
-                    style='{',
-                    defaults={'prefix': ''},
                 )
             )
             syslog_handler.setLevel(syslog_log_level)
@@ -327,15 +396,15 @@ def configure_logging(
     if log_file and log_file_log_level != logging.DISABLED:
         file_handler = logging.handlers.WatchedFileHandler(log_file)
         file_handler.setFormatter(
-            logging.Formatter(
+            Log_prefix_formatter(
                 log_file_format or '[{asctime}] {levelname}: {prefix}{message}',  # noqa: FS003
-                style='{',
-                defaults={'prefix': ''},
             )
         )
         file_handler.setLevel(log_file_log_level)
         handlers.append(file_handler)
 
+    flush_delayed_logging(handlers)
+
     logging.basicConfig(
         level=min(handler.level for handler in handlers),
         handlers=handlers,

docs/_includes/layouts/base.njk

@@ -4,7 +4,7 @@
 		<meta charset="utf-8">
 		<meta name="viewport" content="width=device-width, initial-scale=1.0">
                 <meta name="generator" content="{{ eleventy.generator }}">
-		<link rel="icon" href="docs/static/borgmatic.png" type="image/x-icon">
+		<link rel="icon" href="https://torsion.org/borgmatic/docs/static/borgmatic.png" type="image/x-icon">
 		<title>{{ subtitle + ' - ' if subtitle}}{{ title }}</title>
 {%- set css %}
 {% include 'index.css' %}

docs/how-to/monitor-your-backups.md

@@ -710,7 +710,8 @@ zabbix:
         - fail
 ```
 
-This hook requires the Zabbix server be running version 7.0+
+This hook requires the Zabbix server be running version 7.0. ([Support for Zabbix
+7.2+](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/1003) is planned.)
 
 
 ### Authentication methods

docs/how-to/provide-your-passwords.md

@@ -19,6 +19,7 @@ encryption_passphrase: yourpassphrase
 But if you'd rather store them outside of borgmatic, whether for convenience
 or security reasons, read on.
 
+
 ### Delegating to another application
 
 borgmatic supports calling another application such as a password manager to 
@@ -31,71 +32,262 @@ to provide the passphrase:
 encryption_passcommand: pass path/to/borg-passphrase
 ```
 
-Another example for [KeePassXC](https://keepassxc.org/):
-
-```yaml
-encryption_passcommand: keepassxc-cli show --show-protected --attributes Password credentials.kdbx borg_passphrase
-```
-
-... where `borg_passphrase` is the title of the KeePassXC entry containing your
-Borg encryption passphrase in its `Password` field.
-
 <span class="minilink minilink-addedin">New in version 1.9.9</span> Instead of
 letting Borg run the passcommand—potentially multiple times since borgmatic runs
 Borg multiple times—borgmatic now runs the passcommand itself and passes the
-resulting passprhase securely to Borg via an anonymous pipe. This means you
+resulting passphrase securely to Borg via an anonymous pipe. This means you
 should only ever get prompted for your password manager's passphrase at most
 once per borgmatic run.
 
 
-### Using systemd service credentials
+### systemd service credentials
 
-Borgmatic supports using encrypted [credentials](https://systemd.io/CREDENTIALS/).
+borgmatic supports reading encrypted [systemd
+credentials](https://systemd.io/CREDENTIALS/). To use this feature, start by
+saving your password as an encrypted credential to
+`/etc/credstore.encrypted/borgmatic.pw`, e.g.,
 
-Save your password as an encrypted credential to `/etc/credstore.encrypted/borgmatic.pw`, e.g.,
-
-```
-# systemd-ask-password -n | systemd-creds encrypt - /etc/credstore.encrypted/borgmatic.pw
+```bash
+systemd-ask-password -n | systemd-creds encrypt - /etc/credstore.encrypted/borgmatic.pw
 ```
 
-Then uncomment or use the following in your configuration file:
+Then use the following in your configuration file:
 
 ```yaml
-encryption_passcommand: "cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw"
+encryption_passphrase: "{credential systemd borgmatic.pw}"
+```
+
+<span class="minilink minilink-addedin">Prior to version 1.9.10</span> You can
+accomplish the same thing with this configuration:
+
+```yaml
+encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw
 ```
 
 Note that the name `borgmatic.pw` is hardcoded in the systemd service file.
 
-To use multiple different passwords, save them as encrypted credentials to `/etc/credstore.encrypted/borgmatic/`, e.g.,
+The `{credential ...}` syntax works for several different options in a borgmatic
+configuration file besides just `encryption_passphrase`. For instance, the
+username, password, and API token options within database and monitoring hooks
+support `{credential ...}`:
 
+```yaml
+postgresql_databases:
+    - name: invoices
+      username: postgres
+      password: "{credential systemd borgmatic_db1}"
 ```
-# mkdir /etc/credstore.encrypted/borgmatic
-# systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_backupserver1 - /etc/credstore.encrypted/borgmatic/backupserver1
-# systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_pw2 - /etc/credstore.encrypted/borgmatic/pw2
+
+For specifics about which options are supported, see the
+[configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/).
+
+To use these credentials, you'll need to modify the borgmatic systemd service
+file to support loading multiple credentials (assuming you need to load more
+than one or anything not named `borgmatic.pw`).
+
+Start by saving each encrypted credentials to
+`/etc/credstore.encrypted/borgmatic/`. E.g.,
+
+```bash
+mkdir /etc/credstore.encrypted/borgmatic
+systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_backupserver1 - /etc/credstore.encrypted/borgmatic/backupserver1
+systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_pw2 - /etc/credstore.encrypted/borgmatic/pw2
 ...
 ```
 
-Ensure that the file names, (e.g. `backupserver1`) match the corresponding part of
-the `--name` option *after* the underscore (_), and that the part *before* 
+Ensure that the file names, (e.g. `backupserver1`) match the corresponding part
+of the `--name` option *after* the underscore (_), and that the part *before*
 the underscore matches the directory name (e.g. `borgmatic`).
 
 Then, uncomment the appropriate line in the systemd service file:
 
 ```
-# systemctl edit borgmatic.service
+systemctl edit borgmatic.service
 ...
 # Load multiple encrypted credentials.
 LoadCredentialEncrypted=borgmatic:/etc/credstore.encrypted/borgmatic/
 ```
 
-Finally, use the following in your configuration file:
+Finally, use something like the following in your borgmatic configuration file
+for each option value you'd like to load from systemd:
 
-```
-encryption_passcommand: "cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1"
+```yaml
+encryption_passphrase: "{credential systemd borgmatic_backupserver1}"
 ```
 
-Adjust `borgmatic_backupserver1` according to the name given to the credential 
-and the directory set in the service file.
+<span class="minilink minilink-addedin">Prior to version 1.9.10</span> Use the
+following instead, but only for the `encryption_passcommand` option and
+not other options:
+
+```yaml
+encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1
+```
+
+Adjust `borgmatic_backupserver1` according to the name of the credential and the
+directory set in the service file.
+
+Be aware that when using this systemd `{credential ...}` feature, you may no
+longer be able to run certain borgmatic actions outside of the systemd service,
+as the credentials are only available from within the context of that service.
+So for instance, `borgmatic list` necessarily relies on the
+`encryption_passphrase` in order to access the Borg repository, but `list`
+shouldn't need to load any credentials for your database or monitoring hooks.
+
+The one exception is `borgmatic config validate`, which doesn't actually load
+any credentials and should continue working anywhere.
+
+
+### Container secrets
+
+<span class="minilink minilink-addedin">New in version 1.9.11</span> When
+running inside a container, borgmatic can read [Docker
+secrets](https://docs.docker.com/compose/how-tos/use-secrets/) and [Podman
+secrets](https://www.redhat.com/en/blog/new-podman-secrets-command). Creating
+those secrets and passing them into your borgmatic container is outside the
+scope of this documentation, but here's a simple example of that with [Docker
+Compose](https://docs.docker.com/compose/):
+
+```yaml
+services:
+  borgmatic:
+    # Use the actual image name of your borgmatic container here.
+    image: borgmatic:latest
+    secrets:
+      - borgmatic_passphrase
+secrets:
+  borgmatic_passphrase:
+    file: /etc/borgmatic/passphrase.txt
+```
+
+This assumes there's a file on the host at `/etc/borgmatic/passphrase.txt`
+containing your passphrase. Docker or Podman mounts the contents of that file
+into a secret named `borgmatic_passphrase` in the borgmatic container at
+`/run/secrets/`.
+
+Once your container secret is in place, you can consume it within your borgmatic
+configuration file:
+
+```yaml
+encryption_passphrase: "{credential container borgmatic_passphrase}"
+```
+
+This reads the secret securely from a file mounted at
+`/run/secrets/borgmatic_passphrase` within the borgmatic container.
+
+The `{credential ...}` syntax works for several different options in a borgmatic
+configuration file besides just `encryption_passphrase`. For instance, the
+username, password, and API token options within database and monitoring hooks
+support `{credential ...}`:
+
+```yaml
+postgresql_databases:
+    - name: invoices
+      username: postgres
+      password: "{credential container borgmatic_db1}"
+```
+
+For specifics about which options are supported, see the
+[configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/).
+
+You can also optionally override the `/run/secrets` directory that borgmatic reads secrets from
+inside a container:
+
+```yaml
+container:
+    secrets_directory: /path/to/secrets
+```
+
+But you should only need to do this for development or testing purposes.
+
+
+### KeePassXC passwords
+
+<span class="minilink minilink-addedin">New in version 1.9.11</span> borgmatic
+supports reading passwords from the [KeePassXC](https://keepassxc.org/) password
+manager. To use this feature, start by creating an entry in your KeePassXC
+database, putting your password into the "Password" field of that entry and
+making sure it's saved.
+
+Then, you can consume that password in your borgmatic configuration file. For
+instance, if the entry's title is "borgmatic" and your KeePassXC database is
+located at `/etc/keys.kdbx`, do this:
+
+```yaml
+encryption_passphrase: "{credential keepassxc /etc/keys.kdbx borgmatic}"
+```
+
+But if the entry's title is multiple words like `borg pw`, you'll
+need to quote it:
+
+```yaml
+encryption_passphrase: "{credential keepassxc /etc/keys.kdbx 'borg pw'}"
+```
+
+With this in place, borgmatic runs the `keepassxc-cli` command to retrieve the
+passphrase on demand. But note that `keepassxc-cli` will prompt for its own
+passphrase in order to unlock its database, so be prepared to enter it when
+running borgmatic.
+
+The `{credential ...}` syntax works for several different options in a borgmatic
+configuration file besides just `encryption_passphrase`. For instance, the
+username, password, and API token options within database and monitoring hooks
+support `{credential ...}`:
+
+```yaml
+postgresql_databases:
+    - name: invoices
+      username: postgres
+      password: "{credential keepassxc /etc/keys.kdbx database}"
+```
+
+For specifics about which options are supported, see the
+[configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/).
+
+You can also optionally override the `keepassxc-cli` command that borgmatic calls to load
+passwords:
+
+```yaml
+keepassxc:
+    keepassxc_cli_command: /usr/local/bin/keepassxc-cli
+```
+
+
+### File-based credentials
+
+<span class="minilink minilink-addedin">New in version 1.9.11</span> borgmatic
+supports reading credentials from arbitrary file paths. To use this feature,
+start by writing your credential into a file that borgmatic has permission to
+read. Take care not to include anything in the file other than your credential.
+(borgmatic is smart enough to strip off a trailing newline though.)
+
+You can consume that credential file in your borgmatic configuration. For
+instance, if your credential file is at `/credentials/borgmatic.txt`, do this:
+
+```yaml
+encryption_passphrase: "{credential file /credentials/borgmatic.txt}"
+```
+
+With this in place, borgmatic reads the credential from the file path.
+
+The `{credential ...}` syntax works for several different options in a borgmatic
+configuration file besides just `encryption_passphrase`. For instance, the
+username, password, and API token options within database and monitoring hooks
+support `{credential ...}`:
+
+```yaml
+postgresql_databases:
+    - name: invoices
+      username: postgres
+      password: "{credential file /credentials/database.txt}"
+```
+
+For specifics about which options are supported, see the
+[configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/).
+
 
 ### Environment variable interpolation
 
@@ -103,7 +295,15 @@ and the directory set in the service file.
 supports interpolating arbitrary environment variables directly into option
 values in your configuration file. That means you can instruct borgmatic to
 pull your repository passphrase, your database passwords, or any other option
-values from environment variables. For instance:
+values from environment variables.
+
+Be aware though that environment variables may be less secure than some of the
+other approaches above for getting credentials into borgmatic. That's because
+environment variables may be visible from within child processes and/or OS-level
+process metadata.
+
+Here's an example of using an environment variable from borgmatic's
+configuration file:
 
 ```yaml
 encryption_passphrase: ${YOUR_PASSPHRASE}
@@ -165,6 +365,7 @@ can escape it with a backslash. For instance, if your password is literally
 encryption_passphrase: \${A}@!
 ```
 
+
 ## Related features
 
 Another way to override particular options within a borgmatic configuration
@@ -177,9 +378,3 @@ Additionally, borgmatic action hooks support their own [variable
 interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation),
 although in that case it's for particular borgmatic runtime values rather than
 (only) environment variables.
-
-Lastly, if you do want to specify your passhprase directly within borgmatic
-configuration, but you'd like to keep it in a separate file from your main
-configuration, you can [use a configuration include or a merge
-include](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-includes)
-to pull in an external password.

docs/how-to/snapshot-your-filesystems.md

@@ -54,8 +54,8 @@ You have a couple of options for borgmatic to find and backup your ZFS datasets:
  * For any dataset you'd like backed up, add its mount point to borgmatic's
    `source_directories` option.
  * <span class="minilink minilink-addedin">New in version 1.9.6</span> Or
-   include the mount point with borgmatic's `patterns` or `patterns_from`
-   options.
+   include the mount point as a root pattern with borgmatic's `patterns` or
+   `patterns_from` options.
  * Or set the borgmatic-specific user property
    `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs
    set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find
@@ -65,6 +65,11 @@ If you have multiple borgmatic configuration files with ZFS enabled, and you'd
 like particular datasets to be backed up only for particular configuration
 files, use the `source_directories` option instead of the user property.
 
+<span class="minilink minilink-addedin">New in version 1.9.11</span> borgmatic
+won't snapshot datasets with the `canmount=off` property, which is often set on
+datasets that only serve as a container for other datasets. Use `zfs get
+canmount datasetname` to see the `canmount` value for a dataset.
+
 During a backup, borgmatic automatically snapshots these discovered datasets
 (non-recursively), temporarily mounts the snapshots within its [runtime
 directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory),
@@ -147,7 +152,8 @@ For any subvolume you'd like backed up, add its path to borgmatic's
 `source_directories` option.
 
 <span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
-the mount point with borgmatic's `patterns` or `patterns_from` options.
+the mount point as a root pattern with borgmatic's `patterns` or `patterns_from`
+options.
 
 During a backup, borgmatic snapshots these subvolumes (non-recursively) and
 includes the snapshotted files in the paths sent to Borg. borgmatic is also
@@ -252,7 +258,8 @@ For any logical volume you'd like backed up, add its mount point to
 borgmatic's `source_directories` option.
 
 <span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
-the mount point with borgmatic's `patterns` or `patterns_from` options.
+the mount point as a root pattern with borgmatic's `patterns` or `patterns_from`
+options.
 
 During a backup, borgmatic automatically snapshots these discovered logical volumes
 (non-recursively), temporarily mounts the snapshots within its [runtime

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "1.9.9"
+version = "1.9.11"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]

scripts/run-full-tests

@@ -18,8 +18,7 @@ if [ -z "$TEST_CONTAINER" ]; then
 fi
 
 apk add --no-cache python3 py3-pip borgbackup postgresql17-client mariadb-client mongodb-tools \
-    py3-mongo py3-regex py3-ruamel.yaml py3-ruamel.yaml.clib py3-tox py3-typed-ast py3-yaml bash \
-    sqlite fish
+    py3-mongo py3-regex py3-ruamel.yaml py3-ruamel.yaml.clib py3-tox py3-yaml bash sqlite fish
 export COVERAGE_FILE=/tmp/.coverage
 
 tox --workdir /tmp/.tox --sitepackages

test_requirements.txt

@@ -30,4 +30,3 @@ regex
 requests==2.32.2
 ruamel.yaml>0.15.0
 toml==0.10.2
-typed-ast

tests/end-to-end/commands/fake_keepassxc_cli.py

@@ -0,0 +1,29 @@
+import argparse
+import sys
+
+
+def parse_arguments(*unparsed_arguments):
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument('command')
+    parser.add_argument('--show-protected', action='store_true')
+    parser.add_argument('--attributes')
+    parser.add_argument('database_path')
+    parser.add_argument('attribute_name')
+
+    return parser.parse_args(unparsed_arguments)
+
+
+def main():
+    arguments = parse_arguments(*sys.argv[1:])
+
+    assert arguments.command == 'show'
+    assert arguments.show_protected
+    assert arguments.attributes == 'Password'
+    assert arguments.database_path.endswith('.kdbx')
+    assert arguments.attribute_name
+
+    print('test')
+
+
+if __name__ == '__main__':
+    main()

tests/end-to-end/commands/fake_zfs.py

@@ -27,6 +27,7 @@ BUILTIN_DATASETS = (
         'used': '256K',
         'avail': '23.7M',
         'refer': '25K',
+        'canmount': 'on',
         'mountpoint': '/pool',
     },
     {
@@ -34,6 +35,7 @@ BUILTIN_DATASETS = (
         'used': '256K',
         'avail': '23.7M',
         'refer': '25K',
+        'canmount': 'on',
         'mountpoint': '/pool/dataset',
     },
 )

tests/end-to-end/hooks/credential/test_container.py

@@ -0,0 +1,68 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path, secrets_directory):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase loaded from container secrets in the given secrets
+    directory.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "{credential container mysecret}"'
+        + f'\ncontainer:\n    secrets_directory: {secrets_directory}'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_container_secret():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    os.chdir(temporary_directory)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path, secrets_directory=temporary_directory)
+
+        secret_path = os.path.join(temporary_directory, 'mysecret')
+        with open(secret_path, 'w') as secret_file:
+            secret_file.write('test')
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/end-to-end/hooks/credential/test_file.py

@@ -0,0 +1,68 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path, credential_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase loaded from file at the given credential path.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "{credential file '
+        + credential_path
+        + '}"'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_file_credential():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    os.chdir(temporary_directory)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        credential_path = os.path.join(temporary_directory, 'mycredential')
+        generate_configuration(config_path, repository_path, credential_path)
+
+        with open(credential_path, 'w') as credential_file:
+            credential_file.write('test')
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/end-to-end/hooks/credential/test_keepassxc.py

@@ -0,0 +1,67 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase loaded from keepassxc-cli.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "{credential keepassxc keys.kdbx mypassword}"'
+        + '\nkeepassxc:\n    keepassxc_cli_command: python3 /app/tests/end-to-end/commands/fake_keepassxc_cli.py'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_keepassxc_password():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    os.chdir(temporary_directory)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path)
+
+        database_path = os.path.join(temporary_directory, 'keys.kdbx')
+        with open(database_path, 'w') as database_file:
+            database_file.write('fake KeePassXC database to pacify file existence check')
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/end-to-end/hooks/credential/test_systemd.py

@@ -0,0 +1,69 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase loaded from systemd.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "{credential systemd mycredential}"'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_systemd_credential():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    os.chdir(temporary_directory)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path)
+
+        credential_path = os.path.join(temporary_directory, 'mycredential')
+        with open(credential_path, 'w') as credential_file:
+            credential_file.write('test')
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+            env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}),
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+            env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}),
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+            env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}),
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/unit/actions/test_create.py

@@ -5,16 +5,21 @@ import pytest
 from flexmock import flexmock
 
 from borgmatic.actions import create as module
-from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
+from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
 
 
 @pytest.mark.parametrize(
     'pattern_line,expected_pattern',
     (
-        ('R /foo', Pattern('/foo')),
-        ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)),
-        ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)),
-        ('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)),
+        ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)),
+        ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)),
+        ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)),
+        (
+            '+ sh:/foo*',
+            Pattern(
+                '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG
+            ),
+        ),
     ),
 )
 def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern):
@@ -28,8 +33,8 @@ def test_parse_pattern_with_invalid_pattern_line_errors():
 
 def test_collect_patterns_converts_source_directories():
     assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == (
-        Pattern('/foo'),
-        Pattern('/bar'),
+        Pattern('/foo', source=Pattern_source.CONFIG),
+        Pattern('/bar', source=Pattern_source.CONFIG),
     )
 
 
@@ -48,9 +53,15 @@ def test_collect_patterns_parses_config_patterns():
 
 def test_collect_patterns_converts_exclude_patterns():
     assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
-        Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
-        Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
-        Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL),
+        Pattern(
+            '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
+        ),
+        Pattern(
+            '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
+        ),
+        Pattern(
+            '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG
+        ),
     )
 
 

tests/unit/borg/test_borg.py

@@ -17,7 +17,7 @@ def test_run_arbitrary_borg_calls_borg_with_flags():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -41,7 +41,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
         ('borg', 'break-lock', '--info', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -66,7 +66,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
         ('borg', 'break-lock', '--debug', '--show-rc', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -94,7 +94,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
         ('borg', 'break-lock', '--lock-wait', '5', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -118,7 +118,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
         ('borg', 'break-lock', "'::$ARCHIVE'"),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -143,7 +143,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
         ('borg1', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg1',
         borg_exit_codes=None,
@@ -169,7 +169,7 @@ def test_run_arbitrary_borg_with_exit_codes_calls_borg_using_them():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -195,7 +195,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags()
         ('borg', 'break-lock', '--remote-path', 'borg1', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -222,7 +222,7 @@ def test_run_arbitrary_borg_with_remote_path_injection_attack_gets_escaped():
         ('borg', 'break-lock', '--remote-path', "'borg1; naughty-command'", '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -247,7 +247,7 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg():
         ('borg', 'list', '--progress', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -271,7 +271,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -295,7 +295,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise():
         ('borg',),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -319,7 +319,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags
         ('borg', 'key', 'export', '--info', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -344,7 +344,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla
         ('borg', 'debug', 'dump-manifest', '--info', '::', 'path'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -371,7 +371,7 @@ def test_run_arbitrary_borg_calls_borg_with_working_directory():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_break_lock.py

@@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes
     )
     flexmock(module).should_receive('execute_command').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_change_passphrase.py

@@ -25,7 +25,7 @@ def insert_execute_command_mock(
         command,
         output_file=output_file,
         output_log_level=module.logging.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_check.py

@@ -8,14 +8,17 @@ from borgmatic.borg import check as module
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_execute_command_mock(command, working_directory=None, borg_exit_codes=None):
+def insert_execute_command_mock(
+    command, output_file=None, working_directory=None, borg_exit_codes=None
+):
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         working_directory,
     )
     flexmock(module).should_receive('execute_command').with_args(
         command,
-        extra_environment=None,
+        output_file=output_file,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,
@@ -250,11 +253,11 @@ def test_make_check_name_flags_without_archives_check_and_with_archive_filter_fl
     assert flags == ('--repository-only',)
 
 
-def test_make_check_name_flags_with_data_check_returns_flag_and_implies_archives():
+def test_make_check_name_flags_with_archives_and_data_check_returns_verify_data_flag():
     flexmock(module.feature).should_receive('available').and_return(True)
     flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
 
-    flags = module.make_check_name_flags({'data'}, ())
+    flags = module.make_check_name_flags({'archives', 'data'}, ())
 
     assert flags == (
         '--archives-only',
@@ -262,6 +265,15 @@ def test_make_check_name_flags_with_data_check_returns_flag_and_implies_archives
     )
 
 
+def test_make_check_name_flags_with_repository_and_data_check_returns_verify_data_flag():
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
+
+    flags = module.make_check_name_flags({'archives', 'data', 'repository'}, ())
+
+    assert flags == ('--verify-data',)
+
+
 def test_make_check_name_flags_with_extract_omits_extract_flag():
     flexmock(module.feature).should_receive('available').and_return(True)
     flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
@@ -271,21 +283,6 @@ def test_make_check_name_flags_with_extract_omits_extract_flag():
     assert flags == ()
 
 
-def test_make_check_name_flags_with_repository_and_data_checks_does_not_return_repository_only():
-    flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
-
-    flags = module.make_check_name_flags(
-        {
-            'repository',
-            'data',
-        },
-        (),
-    )
-
-    assert flags == ('--verify-data',)
-
-
 def test_get_repository_id_with_valid_json_does_not_raise():
     config = {}
     flexmock(module.repo_info).should_receive('display_repository_info').and_return(
@@ -336,14 +333,16 @@ def test_get_repository_id_with_missing_json_keys_raises():
 
 def test_check_archives_with_progress_passes_through_to_borg():
     config = {}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--progress', 'repo'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -369,14 +368,16 @@ def test_check_archives_with_progress_passes_through_to_borg():
 
 def test_check_archives_with_repair_passes_through_to_borg():
     config = {}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--repair', 'repo'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -402,13 +403,16 @@ def test_check_archives_with_repair_passes_through_to_borg():
 
 def test_check_archives_with_max_duration_flag_passes_through_to_borg():
     config = {}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--max-duration', '33', 'repo'),
-        extra_environment=None,
+        output_file=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -432,38 +436,18 @@ def test_check_archives_with_max_duration_flag_passes_through_to_borg():
     )
 
 
-def test_check_archives_with_max_duration_flag_and_archives_check_errors():
-    config = {}
-    flexmock(module).should_receive('execute_command').never()
-
-    with pytest.raises(ValueError):
-        module.check_archives(
-            repository_path='repo',
-            config=config,
-            local_borg_version='1.2.3',
-            check_arguments=flexmock(
-                progress=None,
-                repair=None,
-                only_checks=None,
-                force=None,
-                match_archives=None,
-                max_duration=33,
-            ),
-            global_arguments=flexmock(log_json=False),
-            checks={'repository', 'archives'},
-            archive_filter_flags=(),
-        )
-
-
 def test_check_archives_with_max_duration_option_passes_through_to_borg():
     config = {'checks': [{'name': 'repository', 'max_duration': 33}]}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--max-duration', '33', 'repo'),
-        extra_environment=None,
+        output_file=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -487,38 +471,146 @@ def test_check_archives_with_max_duration_option_passes_through_to_borg():
     )
 
 
-def test_check_archives_with_max_duration_option_and_archives_check_errors():
-    config = {'checks': [{'name': 'repository', 'max_duration': 33}]}
-    flexmock(module).should_receive('execute_command').never()
+def test_check_archives_with_max_duration_option_and_archives_check_runs_repository_check_separately():
+    config = {'checks': [{'name': 'repository', 'max_duration': 33}, {'name': 'archives'}]}
+    flexmock(module).should_receive('make_check_name_flags').with_args({'archives'}, ()).and_return(
+        ('--archives-only',)
+    )
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(('--repository-only',))
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    insert_execute_command_mock(('borg', 'check', '--archives-only', 'repo'))
+    insert_execute_command_mock(
+        ('borg', 'check', '--max-duration', '33', '--repository-only', 'repo')
+    )
 
-    with pytest.raises(ValueError):
-        module.check_archives(
-            repository_path='repo',
-            config=config,
-            local_borg_version='1.2.3',
-            check_arguments=flexmock(
-                progress=None,
-                repair=None,
-                only_checks=None,
-                force=None,
-                match_archives=None,
-                max_duration=None,
-            ),
-            global_arguments=flexmock(log_json=False),
-            checks={'repository', 'archives'},
-            archive_filter_flags=(),
-        )
+    module.check_archives(
+        repository_path='repo',
+        config=config,
+        local_borg_version='1.2.3',
+        check_arguments=flexmock(
+            progress=None,
+            repair=None,
+            only_checks=None,
+            force=None,
+            match_archives=None,
+            max_duration=None,
+        ),
+        global_arguments=flexmock(log_json=False),
+        checks={'repository', 'archives'},
+        archive_filter_flags=(),
+    )
+
+
+def test_check_archives_with_max_duration_flag_and_archives_check_runs_repository_check_separately():
+    config = {'checks': [{'name': 'repository'}, {'name': 'archives'}]}
+    flexmock(module).should_receive('make_check_name_flags').with_args({'archives'}, ()).and_return(
+        ('--archives-only',)
+    )
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(('--repository-only',))
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    insert_execute_command_mock(('borg', 'check', '--archives-only', 'repo'))
+    insert_execute_command_mock(
+        ('borg', 'check', '--max-duration', '33', '--repository-only', 'repo')
+    )
+
+    module.check_archives(
+        repository_path='repo',
+        config=config,
+        local_borg_version='1.2.3',
+        check_arguments=flexmock(
+            progress=None,
+            repair=None,
+            only_checks=None,
+            force=None,
+            match_archives=None,
+            max_duration=33,
+        ),
+        global_arguments=flexmock(log_json=False),
+        checks={'repository', 'archives'},
+        archive_filter_flags=(),
+    )
+
+
+def test_check_archives_with_max_duration_option_and_data_check_runs_repository_check_separately():
+    config = {'checks': [{'name': 'repository', 'max_duration': 33}, {'name': 'data'}]}
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'data', 'archives'}, ()
+    ).and_return(('--archives-only', '--verify-data'))
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(('--repository-only',))
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    insert_execute_command_mock(('borg', 'check', '--archives-only', '--verify-data', 'repo'))
+    insert_execute_command_mock(
+        ('borg', 'check', '--max-duration', '33', '--repository-only', 'repo')
+    )
+
+    module.check_archives(
+        repository_path='repo',
+        config=config,
+        local_borg_version='1.2.3',
+        check_arguments=flexmock(
+            progress=None,
+            repair=None,
+            only_checks=None,
+            force=None,
+            match_archives=None,
+            max_duration=None,
+        ),
+        global_arguments=flexmock(log_json=False),
+        checks={'repository', 'data'},
+        archive_filter_flags=(),
+    )
+
+
+def test_check_archives_with_max_duration_flag_and_data_check_runs_repository_check_separately():
+    config = {'checks': [{'name': 'repository'}, {'name': 'data'}]}
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'data', 'archives'}, ()
+    ).and_return(('--archives-only', '--verify-data'))
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(('--repository-only',))
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    insert_execute_command_mock(('borg', 'check', '--archives-only', '--verify-data', 'repo'))
+    insert_execute_command_mock(
+        ('borg', 'check', '--max-duration', '33', '--repository-only', 'repo')
+    )
+
+    module.check_archives(
+        repository_path='repo',
+        config=config,
+        local_borg_version='1.2.3',
+        check_arguments=flexmock(
+            progress=None,
+            repair=None,
+            only_checks=None,
+            force=None,
+            match_archives=None,
+            max_duration=33,
+        ),
+        global_arguments=flexmock(log_json=False),
+        checks={'repository', 'data'},
+        archive_filter_flags=(),
+    )
 
 
 def test_check_archives_with_max_duration_flag_overrides_max_duration_option():
     config = {'checks': [{'name': 'repository', 'max_duration': 33}]}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--max-duration', '44', 'repo'),
-        extra_environment=None,
+        output_file=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -575,9 +667,37 @@ def test_check_archives_calls_borg_with_parameters(checks):
     )
 
 
+def test_check_archives_with_data_check_implies_archives_check_calls_borg_with_parameters():
+    config = {}
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'data', 'archives'}, ()
+    ).and_return(())
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    insert_execute_command_mock(('borg', 'check', 'repo'))
+
+    module.check_archives(
+        repository_path='repo',
+        config=config,
+        local_borg_version='1.2.3',
+        check_arguments=flexmock(
+            progress=None,
+            repair=None,
+            only_checks=None,
+            force=None,
+            match_archives=None,
+            max_duration=None,
+        ),
+        global_arguments=flexmock(log_json=False),
+        checks={'data'},
+        archive_filter_flags=(),
+    )
+
+
 def test_check_archives_with_log_info_passes_through_to_borg():
     config = {}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     insert_logging_mock(logging.INFO)
     insert_execute_command_mock(('borg', 'check', '--info', 'repo'))
@@ -602,7 +722,9 @@ def test_check_archives_with_log_info_passes_through_to_borg():
 
 def test_check_archives_with_log_debug_passes_through_to_borg():
     config = {}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     insert_logging_mock(logging.DEBUG)
     insert_execute_command_mock(('borg', 'check', '--debug', '--show-rc', 'repo'))
@@ -806,7 +928,9 @@ def test_check_archives_with_retention_prefix():
 
 def test_check_archives_with_extra_borg_options_passes_through_to_borg():
     config = {'extra_borg_options': {'check': '--extra --options'}}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo'))
 
@@ -829,16 +953,17 @@ def test_check_archives_with_extra_borg_options_passes_through_to_borg():
 
 
 def test_check_archives_with_match_archives_passes_through_to_borg():
-    config = {}
-    flexmock(module).should_receive('make_check_name_flags').and_return(
-        ('--match-archives', 'foo-*')
-    )
+    config = {'checks': [{'name': 'archives'}]}
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'archives'}, object
+    ).and_return(('--match-archives', 'foo-*'))
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--match-archives', 'foo-*', 'repo'),
-        extra_environment=None,
+        output_file=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -864,7 +989,9 @@ def test_check_archives_with_match_archives_passes_through_to_borg():
 
 def test_check_archives_calls_borg_with_working_directory():
     config = {'working_directory': '/working/dir'}
-    flexmock(module).should_receive('make_check_name_flags').and_return(())
+    flexmock(module).should_receive('make_check_name_flags').with_args(
+        {'repository'}, ()
+    ).and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)

tests/unit/borg/test_compact.py

@@ -17,7 +17,7 @@ def insert_execute_command_mock(
     flexmock(module).should_receive('execute_command').with_args(
         compact_command,
         output_log_level=output_log_level,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=compact_command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_create.py

@@ -4,7 +4,7 @@ import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import create as module
-from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
+from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
 
 from ..test_verbosity import insert_logging_mock
 
@@ -185,6 +185,13 @@ def test_any_parent_directories_treats_unrelated_paths_as_non_match():
 
 
 def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_list():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         'Processing files ...\n- /foo\n+ /bar\n- /baz'
     )
@@ -198,12 +205,18 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/bar', '/baz')
 
 
 def test_collect_special_file_paths_skips_borgmatic_runtime_directory():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /run/borgmatic/bar\n- /baz'
     )
@@ -225,12 +238,18 @@ def test_collect_special_file_paths_skips_borgmatic_runtime_directory():
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/baz')
 
 
 def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_from_paths_output_errors():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /bar\n- /baz'
     )
@@ -245,12 +264,18 @@ def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_fro
             config={},
             local_path=None,
             working_directory=None,
-            borg_environment=None,
             borgmatic_runtime_directory='/run/borgmatic',
         )
 
 
 def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory_missing_from_paths_output_does_not_raise():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /bar\n- /baz'
     )
@@ -264,12 +289,18 @@ def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/bar', '/baz')
 
 
 def test_collect_special_file_paths_excludes_non_special_files():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n+ /bar\n+ /baz'
     )
@@ -285,35 +316,10 @@ def test_collect_special_file_paths_excludes_non_special_files():
         config={},
         local_path=None,
         working_directory=None,
-        borg_environment=None,
         borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/baz')
 
 
-def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command():
-    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
-        ('borg', 'create', '--dry-run', '--list'),
-        capture_stderr=True,
-        working_directory=None,
-        extra_environment=None,
-        borg_local_path='borg',
-        borg_exit_codes=None,
-    ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once()
-    flexmock(module).should_receive('special_file').and_return(True)
-    flexmock(module.os.path).should_receive('exists').and_return(False)
-    flexmock(module).should_receive('any_parent_directories').never()
-
-    module.collect_special_file_paths(
-        dry_run=False,
-        create_command=('borg', 'create', '--exclude-nodump'),
-        config={},
-        local_path='borg',
-        working_directory=None,
-        borg_environment=None,
-        borgmatic_runtime_directory='/run/borgmatic',
-    )
-
-
 DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
 REPO_ARCHIVE = (f'repo::{DEFAULT_ARCHIVE_NAME}',)
 
@@ -663,6 +669,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
                 '/dev/null',
                 Pattern_type.NO_RECURSE,
                 Pattern_style.FNMATCH,
+                source=Pattern_source.INTERNAL,
             ),
         ),
         '/run/borgmatic',
@@ -713,6 +720,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
                 '/dev/null',
                 Pattern_type.NO_RECURSE,
                 Pattern_style.FNMATCH,
+                source=Pattern_source.INTERNAL,
             ),
         ),
         '/run/borgmatic',
@@ -969,7 +977,7 @@ def test_create_archive_calls_borg_with_parameters():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1003,7 +1011,7 @@ def test_create_archive_calls_borg_with_environment():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=environment,
+        environment=environment,
     )
 
     module.create_archive(
@@ -1036,7 +1044,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1066,7 +1074,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     )
@@ -1103,7 +1111,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -1133,7 +1141,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     )
@@ -1172,7 +1180,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1209,7 +1217,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory='/working/dir',
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1244,7 +1252,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them():
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1278,7 +1286,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1316,7 +1324,7 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1350,7 +1358,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1385,7 +1393,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1431,7 +1439,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         create_command,
@@ -1441,7 +1449,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1472,7 +1480,7 @@ def test_create_archive_with_json_calls_borg_with_json_flag():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     ).and_return('[]')
@@ -1506,7 +1514,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     ).and_return('[]')
@@ -1547,7 +1555,7 @@ def test_create_archive_calls_borg_with_working_directory():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory='/working/dir',
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(

tests/unit/borg/test_delete.py

@@ -370,9 +370,9 @@ def test_delete_archives_calls_borg_delete_with_working_directory():
     flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never()
     command = flexmock()
     flexmock(module).should_receive('make_delete_command').and_return(command)
-    extra_environment = flexmock()
+    environment = flexmock()
     flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
-        extra_environment
+        environment
     )
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         '/working/dir'
@@ -380,7 +380,7 @@ def test_delete_archives_calls_borg_delete_with_working_directory():
     flexmock(module.borgmatic.execute).should_receive('execute_command').with_args(
         command,
         output_log_level=logging.ANSWER,
-        extra_environment=extra_environment,
+        environment=environment,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_environment.py

@@ -4,6 +4,12 @@ from borgmatic.borg import environment as module
 
 
 def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_file_descriptor_in_environment():
+    flexmock(module.os).should_receive('environ').and_return(
+        {'USER': 'root', 'BORG_PASSCOMMAND': 'nope'}
+    )
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).and_return(None)
     flexmock(module.borgmatic.borg.passcommand).should_receive(
         'get_passphrase_from_passcommand'
     ).and_return('passphrase')
@@ -14,113 +20,153 @@ def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_fil
 
     environment = module.make_environment({'encryption_passcommand': 'command'})
 
-    assert not environment.get('BORG_PASSCOMMAND')
+    assert environment.get('BORG_PASSPHRASE') is None
+    assert environment.get('BORG_PASSCOMMAND') is None
     assert environment.get('BORG_PASSPHRASE_FD') == '3'
 
 
-def test_make_environment_with_passphrase_should_set_environment():
+def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_in_environment():
+    flexmock(module.os).should_receive('environ').and_return(
+        {'USER': 'root', 'BORG_PASSPHRASE': 'nope', 'BORG_PASSCOMMAND': 'nope'}
+    )
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
     flexmock(module.borgmatic.borg.passcommand).should_receive(
         'get_passphrase_from_passcommand'
     ).and_return(None)
-    flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
+    flexmock(module.os).should_receive('pipe').and_return((3, 4))
+    flexmock(module.os).should_receive('write')
+    flexmock(module.os).should_receive('close')
+    flexmock(module.os).should_receive('set_inheritable')
+
     environment = module.make_environment({'encryption_passphrase': 'pass'})
 
-    assert environment.get('BORG_PASSPHRASE') == 'pass'
+    assert environment.get('BORG_PASSPHRASE') is None
+    assert environment.get('BORG_PASSCOMMAND') is None
+    assert environment.get('BORG_PASSPHRASE_FD') == '3'
+
+
+def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_passphrase_file_descriptor_in_environment():
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    config = {'encryption_passphrase': '{credential systemd pass}'}
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential',
+    ).with_args('{credential systemd pass}', config).and_return('pass')
+    flexmock(module.borgmatic.borg.passcommand).should_receive(
+        'get_passphrase_from_passcommand'
+    ).never()
+    flexmock(module.os).should_receive('pipe').and_return((3, 4))
+    flexmock(module.os).should_receive('write')
+    flexmock(module.os).should_receive('close')
+    flexmock(module.os).should_receive('set_inheritable')
+
+    environment = module.make_environment(config)
+
+    assert environment.get('BORG_PASSPHRASE') is None
+    assert environment.get('BORG_PASSPHRASE_FD') == '3'
 
 
 def test_make_environment_with_ssh_command_should_set_environment():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'ssh_command': 'ssh -C'})
 
     assert environment.get('BORG_RSH') == 'ssh -C'
 
 
 def test_make_environment_without_configuration_sets_certain_environment_variables():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({})
 
     # Default environment variables.
     assert environment == {
+        'USER': 'root',
         'BORG_EXIT_CODES': 'modern',
         'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
         'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
     }
 
 
-def test_make_environment_without_configuration_does_not_set_certain_environment_variables_if_already_set():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+def test_make_environment_without_configuration_passes_through_default_environment_variables_untouched():
+    flexmock(module.os).should_receive('environ').and_return(
+        {
+            'USER': 'root',
+            'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup',
+            'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah',
+        }
+    )
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').with_args(
-        'BORG_RELOCATED_REPO_ACCESS_IS_OK'
-    ).and_return('yup')
-    flexmock(module.os.environ).should_receive('get').with_args(
-        'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'
-    ).and_return('nah')
     environment = module.make_environment({})
 
-    assert environment == {'BORG_EXIT_CODES': 'modern'}
+    assert environment == {
+        'USER': 'root',
+        'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup',
+        'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah',
+        'BORG_EXIT_CODES': 'modern',
+    }
 
 
 def test_make_environment_with_relocated_repo_access_true_should_set_environment_yes():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'relocated_repo_access_is_ok': True})
 
     assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
 
 
 def test_make_environment_with_relocated_repo_access_false_should_set_environment_no():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'relocated_repo_access_is_ok': False})
 
     assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'no'
 
 
 def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environment_YES():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'check_i_know_what_i_am_doing': True})
 
     assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'YES'
 
 
 def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environment_NO():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'check_i_know_what_i_am_doing': False})
 
     assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'NO'
 
 
 def test_make_environment_with_integer_variable_value():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'borg_files_cache_ttl': 40})
+
     assert environment.get('BORG_FILES_CACHE_TTL') == '40'

tests/unit/borg/test_export_key.py

@@ -22,7 +22,7 @@ def insert_execute_command_mock(
         command,
         output_file=output_file,
         output_log_level=module.logging.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_export_tar.py

@@ -23,7 +23,7 @@ def insert_execute_command_mock(
         command,
         output_file=None if capture else module.DO_NOT_CAPTURE,
         output_log_level=output_log_level,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=borg_local_path,
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_extract.py

@@ -12,7 +12,7 @@ def insert_execute_command_mock(command, destination_path=None, borg_exit_codes=
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=destination_path,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,
@@ -587,7 +587,7 @@ def test_extract_archive_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', '--progress', 'repo::archive'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -639,7 +639,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
         ('borg', 'extract', '--stdout', 'repo::archive'),
         output_file=module.subprocess.PIPE,
         run_to_completion=False,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -674,7 +674,7 @@ def test_extract_archive_skips_abspath_for_remote_repository():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', 'server:repo::archive'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_flags.py

@@ -285,3 +285,45 @@ def test_warn_for_aggressive_archive_flags_with_glob_archives_and_json_missing_a
     flexmock(module.logger).should_receive('warning').never()
 
     module.warn_for_aggressive_archive_flags(('borg', '--glob-archives', 'foo*'), '{}')
+
+
+def test_omit_flag_removes_flag_from_arguments():
+    assert module.omit_flag(('borg', 'create', '--flag', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_without_flag_present_passes_through_arguments():
+    assert module.omit_flag(('borg', 'create', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_and_value_removes_flag_and_value_from_arguments():
+    assert module.omit_flag_and_value(
+        ('borg', 'create', '--flag', 'value', '--other'), '--flag'
+    ) == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_and_value_with_equals_sign_removes_flag_and_value_from_arguments():
+    assert module.omit_flag_and_value(('borg', 'create', '--flag=value', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_and_value_without_flag_present_passes_through_arguments():
+    assert module.omit_flag_and_value(('borg', 'create', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )

tests/unit/borg/test_info.py

@@ -514,7 +514,7 @@ def test_display_archives_info_calls_borg_with_working_directory():
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         full_command=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,
@@ -523,7 +523,7 @@ def test_display_archives_info_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         full_command=object,
         output_log_level=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,

tests/unit/borg/test_list.py

@@ -353,7 +353,7 @@ def test_list_archive_calls_borg_with_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -419,7 +419,7 @@ def test_list_archive_calls_borg_with_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg2', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg2',
         borg_exit_codes=None,
@@ -469,7 +469,7 @@ def test_list_archive_calls_borg_using_exit_codes():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -507,7 +507,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -520,7 +520,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive1') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -528,7 +528,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive2') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -576,7 +576,7 @@ def test_list_archive_calls_borg_with_archive():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -699,7 +699,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -759,7 +759,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-list', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -808,7 +808,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--repo', 'repo', 'archive1') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -816,7 +816,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--repo', 'repo', 'archive2') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -875,7 +875,7 @@ def test_list_archive_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_mount.py

@@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes
     )
     flexmock(module).should_receive('execute_command').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,
@@ -262,7 +262,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -337,7 +337,7 @@ def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags
             'repo',
             '/mnt',
         ),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_passcommand.py

@@ -3,33 +3,23 @@ from flexmock import flexmock
 from borgmatic.borg import passcommand as module
 
 
-def test_run_passcommand_with_passphrase_configured_bails():
-    flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never()
-
-    assert (
-        module.run_passcommand('passcommand', passphrase_configured=True, working_directory=None)
-        is None
-    )
-
-
-def test_run_passcommand_without_passphrase_configured_executes_passcommand():
+def test_run_passcommand_does_not_raise():
+    module.run_passcommand.cache_clear()
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
-    ).and_return('passphrase').once()
+    ).and_return('passphrase')
 
-    assert (
-        module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None)
-        == 'passphrase'
-    )
+    assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'
 
 
 def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it():
+    module.run_passcommand.cache_clear()
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         '/working'
     )
-    flexmock(module).should_receive('run_passcommand').with_args(
-        'command', False, '/working'
-    ).and_return('passphrase').once()
+    flexmock(module).should_receive('run_passcommand').with_args('command', '/working').and_return(
+        'passphrase'
+    ).once()
 
     assert (
         module.get_passphrase_from_passcommand(
@@ -39,17 +29,17 @@ def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it():
     )
 
 
-def test_get_passphrase_from_passcommand_with_configured_passphrase_and_passcommand_detects_passphrase():
-    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
-        '/working'
-    )
-    flexmock(module).should_receive('run_passcommand').with_args(
-        'command', True, '/working'
-    ).and_return(None).once()
+def test_get_passphrase_from_passcommand_without_configured_passcommand_bails():
+    flexmock(module).should_receive('run_passcommand').never()
 
-    assert (
-        module.get_passphrase_from_passcommand(
-            {'encryption_passphrase': 'passphrase', 'encryption_passcommand': 'command'},
-        )
-        is None
-    )
+    assert module.get_passphrase_from_passcommand({}) is None
+
+
+def test_run_passcommand_caches_passcommand_after_first_call():
+    module.run_passcommand.cache_clear()
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).and_return('passphrase').once()
+
+    assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'
+    assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'

tests/unit/borg/test_prune.py

@@ -17,7 +17,7 @@ def insert_execute_command_mock(
     flexmock(module).should_receive('execute_command').with_args(
         prune_command,
         output_log_level=output_log_level,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=prune_command[0],
         borg_exit_codes=borg_exit_codes,
@@ -497,7 +497,7 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag
             'repo',
         ),
         output_log_level=logging.INFO,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_repo_create.py

@@ -36,7 +36,7 @@ def insert_repo_create_command_mock(
     flexmock(module).should_receive('execute_command').with_args(
         repo_create_command,
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=repo_create_command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_repo_delete.py

@@ -290,7 +290,7 @@ def test_delete_repository_with_defaults_does_not_capture_output():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
-        extra_environment=object,
+        environment=object,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -319,7 +319,7 @@ def test_delete_repository_with_force_captures_output():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=None,
-        extra_environment=object,
+        environment=object,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -348,7 +348,7 @@ def test_delete_repository_with_cache_only_captures_output():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=None,
-        extra_environment=object,
+        environment=object,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -379,7 +379,7 @@ def test_delete_repository_calls_borg_with_working_directory():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_repo_info.py

@@ -24,7 +24,7 @@ def test_display_repository_info_calls_borg_with_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -33,7 +33,7 @@ def test_display_repository_info_calls_borg_with_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -60,7 +60,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -69,7 +69,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -101,7 +101,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -110,7 +110,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -142,7 +142,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -178,7 +178,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--debug', '--show-rc', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -187,7 +187,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--debug', '--show-rc', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -220,7 +220,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -256,7 +256,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -291,7 +291,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg1', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -300,7 +300,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg1',
         borg_exit_codes=None,
@@ -334,7 +334,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them():
     borg_exit_codes = flexmock()
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -343,7 +343,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -375,7 +375,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--remote-path', 'borg1', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -384,7 +384,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--remote-path', 'borg1', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -417,7 +417,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--umask', '077', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -426,7 +426,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--umask', '077', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -462,7 +462,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--log-json', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -471,7 +471,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--log-json', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -504,7 +504,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--lock-wait', '5', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -513,7 +513,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--lock-wait', '5', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -547,7 +547,7 @@ def test_display_repository_info_calls_borg_with_working_directory():
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -556,7 +556,7 @@ def test_display_repository_info_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_repo_list.py

@@ -39,7 +39,7 @@ def test_resolve_archive_name_calls_borg_with_flags():
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
         borg_local_path='borg',
         borg_exit_codes=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
     ).and_return(expected_archive + '\n')
 
@@ -61,7 +61,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -86,7 +86,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -111,7 +111,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg1',
         borg_exit_codes=None,
@@ -137,7 +137,7 @@ def test_resolve_archive_name_with_exit_codes_calls_borg_using_them():
     borg_exit_codes = flexmock()
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -161,7 +161,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -186,7 +186,7 @@ def test_resolve_archive_name_with_umask_calls_borg_with_umask_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--umask', '077') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -209,7 +209,7 @@ def test_resolve_archive_name_without_archives_raises():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -232,7 +232,7 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -257,7 +257,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -285,7 +285,7 @@ def test_resolve_archive_name_calls_borg_with_working_directory():
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
         borg_local_path='borg',
         borg_exit_codes=None,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
     ).and_return(expected_archive + '\n')
 
@@ -773,7 +773,7 @@ def test_list_repository_calls_borg_with_working_directory():
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         full_command=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,
@@ -782,7 +782,7 @@ def test_list_repository_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         full_command=object,
         output_log_level=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,

tests/unit/borg/test_transfer.py

@@ -21,7 +21,7 @@ def test_transfer_archives_calls_borg_with_flags():
         ('borg', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -55,7 +55,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag():
         ('borg', 'transfer', '--repo', 'repo', '--dry-run'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -86,7 +86,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag():
         ('borg', 'transfer', '--info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -117,7 +117,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag():
         ('borg', 'transfer', '--debug', '--show-rc', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -151,7 +151,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag():
         ('borg', 'transfer', '--match-archives', 'archive', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -184,7 +184,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl
         ('borg', 'transfer', '--match-archives', 'sh:foo*', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -217,7 +217,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv
         ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -248,7 +248,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path():
         ('borg2', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg2',
         borg_exit_codes=None,
@@ -281,7 +281,7 @@ def test_transfer_archives_with_exit_codes_calls_borg_using_them():
         ('borg', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -315,7 +315,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags():
         ('borg', 'transfer', '--remote-path', 'borg2', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -349,7 +349,7 @@ def test_transfer_archives_with_umask_calls_borg_with_umask_flags():
         ('borg', 'transfer', '--umask', '077', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -383,7 +383,7 @@ def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags():
         ('borg', 'transfer', '--log-json', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -418,7 +418,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags():
         ('borg', 'transfer', '--lock-wait', '5', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -449,7 +449,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag():
         ('borg', 'transfer', '--progress', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -484,7 +484,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name):
         ('borg', 'transfer', flag_name, 'value', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -521,7 +521,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla
         ('borg', 'transfer', '--repo', 'repo', '--other-repo', 'other'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -566,7 +566,7 @@ def test_transfer_archives_with_date_based_matching_calls_borg_with_date_based_f
         ),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -605,7 +605,7 @@ def test_transfer_archives_calls_borg_with_working_directory():
         ('borg', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_version.py

@@ -23,7 +23,7 @@ def insert_execute_command_and_capture_output_mock(
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=borg_local_path,
         borg_exit_codes=borg_exit_codes,

tests/unit/commands/test_borgmatic.py

@@ -1114,7 +1114,11 @@ def test_run_actions_runs_multiple_actions_in_argument_order():
     )
 
 
-def test_load_configurations_collects_parsed_configurations_and_logs():
+@pytest.mark.parametrize(
+    'resolve_env',
+    ((True, False),),
+)
+def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env):
     configuration = flexmock()
     other_configuration = flexmock()
     test_expected_logs = [flexmock(), flexmock()]
@@ -1123,7 +1127,12 @@ def test_load_configurations_collects_parsed_configurations_and_logs():
         configuration, ['/tmp/test.yaml'], test_expected_logs
     ).and_return(other_configuration, ['/tmp/other.yaml'], other_expected_logs)
 
-    configs, config_paths, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml')))
+    configs, config_paths, logs = tuple(
+        module.load_configurations(
+            ('test.yaml', 'other.yaml'),
+            resolve_env=resolve_env,
+        )
+    )
 
     assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration}
     assert config_paths == ['/tmp/other.yaml', '/tmp/test.yaml']

tests/unit/hooks/credential/test_container.py

@@ -0,0 +1,74 @@
+import io
+import sys
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks.credential import container as module
+
+
+@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar')))
+def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters):
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=credential_parameters
+        )
+
+
+def test_load_credential_with_invalid_secret_name_raises():
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('this is invalid',)
+        )
+
+
+def test_load_credential_reads_named_secret_from_file():
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/run/secrets/mysecret'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/run/secrets/mysecret').and_return(credential_stream)
+
+    assert (
+        module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',))
+        == 'password'
+    )
+
+
+def test_load_credential_with_custom_secrets_directory_looks_there_for_secret_file():
+    config = {'container': {'secrets_directory': '/secrets'}}
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/secrets/mysecret'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/secrets/mysecret').and_return(credential_stream)
+
+    assert (
+        module.load_credential(
+            hook_config=config['container'], config=config, credential_parameters=('mysecret',)
+        )
+        == 'password'
+    )
+
+
+def test_load_credential_with_custom_secrets_directory_prefixes_it_with_working_directory():
+    config = {'container': {'secrets_directory': 'secrets'}, 'working_directory': '/working'}
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/working/secrets/mysecret'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/working/secrets/mysecret').and_return(
+        credential_stream
+    )
+
+    assert (
+        module.load_credential(
+            hook_config=config['container'], config=config, credential_parameters=('mysecret',)
+        )
+        == 'password'
+    )
+
+
+def test_load_credential_with_file_not_found_error_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/run/secrets/mysecret').and_raise(FileNotFoundError)
+
+    with pytest.raises(ValueError):
+        module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',))