Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1014

NEWS

@@ -1,3 +1,39 @@
+1.9.13
+ * #975: Add a "compression" option to the PostgreSQL database hook.
+ * #1001: Fix a ZFS error during snapshot cleanup.
+ * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes.
+ * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than
+   using an environment variable.
+ * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using
+   "--password" on the command-line!
+ * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit.
+ * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification.
+ * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption
+   between client and server.
+
+1.9.12
+ * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will
+   work with Python 3.9, 3.10, and 3.11 again.
+
+1.9.11
+ * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
+ * #996: Fix the "create" action to omit the repository label prefix from Borg's output when
+   databases are enabled.
+ * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure
+   than using an environment variable.
+ * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file
+   detection.
+ * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from
+   a borgmatic configuration option (e.g. "source_directories")—not from other hooks within
+   borgmatic.
+ * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical
+   volumes.
+ * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property.
+ * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
+   source directories.
+
 1.9.10
  * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic
    configuration files. See the documentation for more information:

README.md

@@ -88,6 +88,9 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 ### Credentials
 
 <a href="https://systemd.io/"><img src="docs/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://www.docker.com/"><img src="docs/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://podman.io/"><img src="docs/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://keepassxc.org/"><img src="docs/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 
 ## Getting started

borgmatic/actions/change_passphrase.py

@@ -16,7 +16,7 @@ def run_change_passphrase(
     remote_path,
 ):
     '''
-    Run the "key change-passprhase" action for the given repository.
+    Run the "key change-passphrase" action for the given repository.
     '''
     if (
         change_passphrase_arguments.repository is None

borgmatic/actions/check.py

@@ -391,7 +391,7 @@ def collect_spot_check_source_paths(
     paths_output = borgmatic.execute.execute_command_and_capture_output(
         create_flags + create_positional_arguments,
         capture_stderr=True,
-        extra_environment=borgmatic.borg.environment.make_environment(config),
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/actions/create.py

@@ -36,6 +36,7 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty
         path,
         borgmatic.borg.pattern.Pattern_type(pattern_type),
         borgmatic.borg.pattern.Pattern_style(pattern_style),
+        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
     )
 
 
@@ -51,7 +52,9 @@ def collect_patterns(config):
     try:
         return (
             tuple(
-                borgmatic.borg.pattern.Pattern(source_directory)
+                borgmatic.borg.pattern.Pattern(
+                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
+                )
                 for source_directory in config.get('source_directories', ())
             )
             + tuple(
@@ -144,6 +147,7 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None):
                         pattern.type,
                         pattern.style,
                         pattern.device,
+                        pattern.source,
                     )
                     for expanded_path in expand_directory(pattern.path, working_directory)
                 )
@@ -178,6 +182,7 @@ def device_map_patterns(patterns, working_directory=None):
                 and os.path.exists(full_path)
                 else None
             ),
+            source=pattern.source,
         )
         for pattern in patterns
         for full_path in (os.path.join(working_directory or '', pattern.path),)

borgmatic/borg/borg.py

@@ -61,7 +61,7 @@ def run_arbitrary_borg(
         tuple(shlex.quote(part) for part in full_command),
         output_file=DO_NOT_CAPTURE,
         shell=True,
-        extra_environment=dict(
+        environment=dict(
             (environment.make_environment(config) or {}),
             **{
                 'BORG_REPO': repository_path,

borgmatic/borg/break_lock.py

@@ -36,7 +36,7 @@ def break_lock(
 
     execute_command(
         full_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/change_passphrase.py

@@ -56,7 +56,7 @@ def change_passphrase(
         full_command,
         output_file=borgmatic.execute.DO_NOT_CAPTURE,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config_without_passphrase),
+        environment=environment.make_environment(config_without_passphrase),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/check.py

@@ -182,7 +182,7 @@ def check_archives(
             output_file=(
                 DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None
             ),
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/compact.py

@@ -49,7 +49,7 @@ def compact_segments(
     execute_command(
         full_command,
         output_log_level=logging.INFO,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/create.py

@@ -132,41 +132,53 @@ def collect_special_file_paths(
     used.
 
     Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
-    its own special files there for database dumps. And if the borgmatic runtime directory is
-    configured to be excluded from the files Borg backs up, error, because this means Borg won't be
-    able to consume any database dumps and therefore borgmatic will hang.
+    its own special files there for database dumps and we don't want those omitted.
+
+    Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
+    plans to backup, that means the user must have excluded the runtime directory (e.g. via
+    "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
+    consume any database dumps and therefore borgmatic will hang when it tries to do so.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
-    # files including any named pipe we've created.
+    # files including any named pipe we've created. And omit "--filter" because that can break the
+    # paths output parsing below such that path lines no longer start with th expected "- ".
     paths_output = execute_command_and_capture_output(
-        tuple(argument for argument in create_command if argument != '--exclude-nodump')
+        flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
         + ('--dry-run', '--list'),
         capture_stderr=True,
         working_directory=working_directory,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
     )
 
+    # These are all the individual files that Borg is planning to backup as determined by the Borg
+    # create dry run above.
     paths = tuple(
         path_line.split(' ', 1)[1]
         for path_line in paths_output.split('\n')
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
-    skip_paths = {}
+
+    # These are the subset of those files that contain the borgmatic runtime directory.
+    paths_containing_runtime_directory = {}
 
     if os.path.exists(borgmatic_runtime_directory):
-        skip_paths = {
+        paths_containing_runtime_directory = {
             path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
         }
 
-        if not skip_paths and not dry_run:
+        # If no paths to backup contain the runtime directory, it must've been excluded.
+        if not paths_containing_runtime_directory and not dry_run:
             raise ValueError(
                 f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
             )
 
     return tuple(
-        path for path in paths if special_file(path, working_directory) if path not in skip_paths
+        path
+        for path in paths
+        if special_file(path, working_directory)
+        if path not in paths_containing_runtime_directory
     )
 
 
@@ -325,6 +337,7 @@ def make_base_create_command(
                         special_file_path,
                         borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
                         borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                        source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
                     )
                     for special_file_path in special_file_paths
                 ),
@@ -409,7 +422,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -417,7 +430,7 @@ def create_archive(
         return execute_command_and_capture_output(
             create_flags + create_positional_arguments,
             working_directory=working_directory,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )
@@ -427,7 +440,7 @@ def create_archive(
             output_log_level,
             output_file,
             working_directory=working_directory,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
         )

borgmatic/borg/delete.py

@@ -128,7 +128,7 @@ def delete_archives(
     borgmatic.execute.execute_command(
         command,
         output_log_level=logging.ANSWER,
-        extra_environment=borgmatic.borg.environment.make_environment(config),
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/environment.py

@@ -10,13 +10,10 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
-    'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
 }
 
-CREDENTIAL_OPTIONS = {'encryption_passphrase'}
-
 DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = {
     'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
     'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
@@ -29,32 +26,55 @@ DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = {
 
 def make_environment(config):
     '''
-    Given a borgmatic configuration dict, return its options converted to a Borg environment
-    variable dict.
+    Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it
+    with a copy of the current environment variables, and return the result.
 
     Do not reuse this environment across multiple Borg invocations, because it can include
     references to resources like anonymous pipes for passphrases—which can only be consumed once.
+
+    Here's how native Borg precedence works for a few of the environment variables:
+
+      1. BORG_PASSPHRASE, if set, is used first.
+      2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set.
+      3. BORG_PASSPHRASE_FD is used only if neither of the above are set.
+
+    In borgmatic, we want to simulate this precedence order, but there are some additional
+    complications. First, values can come from either configuration or from environment variables
+    set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a
+    passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we
+    want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's
+    more secure than using an environment variable (BORG_PASSPHRASE).
     '''
-    environment = {}
+    environment = dict(os.environ)
 
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
         value = config.get(option_name)
 
-        if option_name in CREDENTIAL_OPTIONS and value is not None:
-            value = borgmatic.hooks.credential.parse.resolve_credential(value)
-
         if value is not None:
             environment[environment_variable_name] = str(value)
 
-    passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
+    if 'encryption_passphrase' in config:
+        environment.pop('BORG_PASSPHRASE', None)
+        environment.pop('BORG_PASSCOMMAND', None)
 
-    # If the passcommand produced a passphrase, send it to Borg via an anonymous pipe.
-    if passphrase:
+    if 'encryption_passcommand' in config:
+        environment.pop('BORG_PASSCOMMAND', None)
+
+    passphrase = borgmatic.hooks.credential.parse.resolve_credential(
+        config.get('encryption_passphrase'), config
+    )
+
+    if passphrase is None:
+        passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config)
+
+    # If there's a passphrase (from configuration, from a configured credential, or from a
+    # configured passcommand), send it to Borg via an anonymous pipe.
+    if passphrase is not None:
         read_file_descriptor, write_file_descriptor = os.pipe()
         os.write(write_file_descriptor, passphrase.encode('utf-8'))
         os.close(write_file_descriptor)
 
-        # This, plus subprocess.Popen(..., close_fds=False) in execute.py, is necessary for the Borg
+        # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg
         # child process to inherit the file descriptor.
         os.set_inheritable(read_file_descriptor, True)
         environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor)

borgmatic/borg/export_key.py

@@ -67,7 +67,7 @@ def export_key(
         full_command,
         output_file=output_file,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/export_tar.py

@@ -70,7 +70,7 @@ def export_tar_archive(
         full_command,
         output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
         output_log_level=output_log_level,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/extract.py

@@ -58,7 +58,7 @@ def extract_last_archive_dry_run(
 
     execute_command(
         full_extract_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
@@ -154,7 +154,7 @@ def extract_archive(
         return execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=full_destination_path,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -166,7 +166,7 @@ def extract_archive(
             full_command,
             output_file=subprocess.PIPE,
             run_to_completion=False,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=full_destination_path,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -176,7 +176,7 @@ def extract_archive(
     # if the restore paths don't exist in the archive.
     execute_command(
         full_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=full_destination_path,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

borgmatic/borg/flags.py

@@ -156,3 +156,44 @@ def warn_for_aggressive_archive_flags(json_command, json_output):
         logger.debug(f'Cannot parse JSON output from archive command: {error}')
     except (TypeError, KeyError):
         logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
+
+
+def omit_flag(arguments, flag):
+    '''
+    Given a sequence of Borg command-line arguments, return them with the given (valueless) flag
+    omitted. For instance, if the flag is "--flag" and arguments is:
+
+        ('borg', 'create', '--flag', '--other-flag')
+
+    ... then return:
+
+        ('borg', 'create', '--other-flag')
+    '''
+    return tuple(argument for argument in arguments if argument != flag)
+
+
+def omit_flag_and_value(arguments, flag):
+    '''
+    Given a sequence of Borg command-line arguments, return them with the given flag and its
+    corresponding value omitted. For instance, if the flag is "--flag" and arguments is:
+
+        ('borg', 'create', '--flag', 'value', '--other-flag')
+
+    ... or:
+
+        ('borg', 'create', '--flag=value', '--other-flag')
+
+    ... then return:
+
+        ('borg', 'create', '--other-flag')
+    '''
+    # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two',
+    # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')).
+    # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and
+    # its value.
+    return tuple(
+        argument
+        for (previous_argument, argument) in zip((None,) + arguments, arguments)
+        if flag not in (previous_argument, argument)
+        if not argument.startswith(f'{flag}=')
+    )

borgmatic/borg/info.py

@@ -102,7 +102,7 @@ def display_archives_info(
 
     json_info = execute_command_and_capture_output(
         json_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
@@ -116,7 +116,7 @@ def display_archives_info(
     execute_command(
         main_command,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

borgmatic/borg/list.py

@@ -124,7 +124,7 @@ def capture_archive_listing(
                 local_path,
                 remote_path,
             ),
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
@@ -221,7 +221,7 @@ def list_archive(
                     local_path,
                     remote_path,
                 ),
-                extra_environment=environment.make_environment(config),
+                environment=environment.make_environment(config),
                 working_directory=borgmatic.config.paths.get_working_directory(config),
                 borg_local_path=local_path,
                 borg_exit_codes=borg_exit_codes,
@@ -257,7 +257,7 @@ def list_archive(
         execute_command(
             main_command,
             output_log_level=logging.ANSWER,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=borgmatic.config.paths.get_working_directory(config),
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/mount.py

@@ -66,7 +66,7 @@ def mount_archive(
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
@@ -75,7 +75,7 @@ def mount_archive(
 
     execute_command(
         full_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/passcommand.py

@@ -9,21 +9,14 @@ logger = logging.getLogger(__name__)
 
 
 @functools.cache
-def run_passcommand(passcommand, passphrase_configured, working_directory):
+def run_passcommand(passcommand, working_directory):
     '''
     Run the given passcommand using the given working directory and return the passphrase produced
-    by the command. But bail first if a passphrase is already configured; this mimics Borg's
-    behavior.
+    by the command.
 
     Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
     per borgmatic invocation.
     '''
-    if passcommand and passphrase_configured:
-        logger.warning(
-            'Ignoring the "encryption_passcommand" option because "encryption_passphrase" is set'
-        )
-        return None
-
     return borgmatic.execute.execute_command_and_capture_output(
         shlex.split(passcommand),
         working_directory=working_directory,
@@ -44,7 +37,4 @@ def get_passphrase_from_passcommand(config):
     if not passcommand:
         return None
 
-    passphrase = config.get('encryption_passphrase')
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    return run_passcommand(passcommand, bool(passphrase is not None), working_directory)
+    return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config))

borgmatic/borg/pattern.py

@@ -20,12 +20,31 @@ class Pattern_style(enum.Enum):
     PATH_FULL_MATCH = 'pf'
 
 
+class Pattern_source(enum.Enum):
+    '''
+    Where the pattern came from within borgmatic. This is important because certain use cases (like
+    filesystem snapshotting) only want to consider patterns that the user actually put in a
+    configuration file and not patterns from other sources.
+    '''
+
+    # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories".
+    CONFIG = 'config'
+
+    # The pattern is generated internally within borgmatic, e.g. for special file excludes.
+    INTERNAL = 'internal'
+
+    # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump
+    # directory.
+    HOOK = 'hook'
+
+
 Pattern = collections.namedtuple(
     'Pattern',
-    ('path', 'type', 'style', 'device'),
+    ('path', 'type', 'style', 'device', 'source'),
     defaults=(
         Pattern_type.ROOT,
         Pattern_style.NONE,
         None,
+        Pattern_source.HOOK,
     ),
 )

borgmatic/borg/prune.py

@@ -96,7 +96,7 @@ def prune_archives(
     execute_command(
         full_command,
         output_log_level=output_log_level,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_create.py

@@ -98,7 +98,7 @@ def create_repository(
     execute_command(
         repo_create_command,
         output_file=DO_NOT_CAPTURE,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_delete.py

@@ -88,7 +88,7 @@ def delete_repository(
             if repo_delete_arguments.force or repo_delete_arguments.cache_only
             else borgmatic.execute.DO_NOT_CAPTURE
         ),
-        extra_environment=borgmatic.borg.environment.make_environment(config),
+        environment=borgmatic.borg.environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/repo_info.py

@@ -56,7 +56,7 @@ def display_repository_info(
     if repo_info_arguments.json:
         return execute_command_and_capture_output(
             full_command,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,
@@ -65,7 +65,7 @@ def display_repository_info(
         execute_command(
             full_command,
             output_log_level=logging.ANSWER,
-            extra_environment=environment.make_environment(config),
+            environment=environment.make_environment(config),
             working_directory=working_directory,
             borg_local_path=local_path,
             borg_exit_codes=borg_exit_codes,

borgmatic/borg/repo_list.py

@@ -49,7 +49,7 @@ def resolve_archive_name(
 
     output = execute_command_and_capture_output(
         full_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),
@@ -164,7 +164,7 @@ def list_repository(
 
     json_listing = execute_command_and_capture_output(
         json_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,
@@ -178,7 +178,7 @@ def list_repository(
     execute_command(
         main_command,
         output_log_level=logging.ANSWER,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=working_directory,
         borg_local_path=local_path,
         borg_exit_codes=borg_exit_codes,

borgmatic/borg/transfer.py

@@ -57,7 +57,7 @@ def transfer_archives(
         full_command,
         output_log_level=logging.ANSWER,
         output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/borg/version.py

@@ -21,7 +21,7 @@ def local_borg_version(config, local_path='borg'):
     )
     output = execute_command_and_capture_output(
         full_command,
-        extra_environment=environment.make_environment(config),
+        environment=environment.make_environment(config),
         working_directory=borgmatic.config.paths.get_working_directory(config),
         borg_local_path=local_path,
         borg_exit_codes=config.get('borg_exit_codes'),

borgmatic/config/generate.py

@@ -41,6 +41,7 @@ def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     schema_type = schema.get('type')
     example = schema.get('example')
+
     if example is not None:
         return example
 

borgmatic/config/schema.yaml

@@ -1040,6 +1040,18 @@ properties:
                         individual databases. See the pg_dump documentation for
                         more about formats.
                     example: directory
+                compression:
+                    type: ["string", "integer"]
+                    description: |
+                        Database dump compression level (integer) or method
+                        ("gzip", "lz4", "zstd", or "none") and optional
+                        colon-separated detail. Defaults to moderate "gzip" for
+                        "custom" and "directory" formats and no compression for
+                        the "plain" format. Compression is not supported for the
+                        "tar" format. Be aware that Borg does its own
+                        compression as well, so you may not need it in both
+                        places.
+                    example: none
                 ssl_mode:
                     type: string
                     enum: ['disable', 'allow', 'prefer',
@@ -1198,6 +1210,20 @@ properties:
                         Defaults to the "password" option. Supports the
                         "{credential ...}" syntax.
                     example: trustsome1
+                tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and server. The default varies based on the
+                        MariaDB version.
+                    example: false
+                restore_tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and restore server. The default varies based on
+                        the MariaDB version.
+                    example: false
                 mariadb_dump_command:
                     type: string
                     description: |
@@ -1328,6 +1354,20 @@ properties:
                         Defaults to the "password" option. Supports the
                         "{credential ...}" syntax.
                     example: trustsome1
+                tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and server. The default varies based on the
+                        MySQL installation.
+                    example: false
+                restore_tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and restore server. The default varies based on
+                        the MySQL installation.
+                    example: false
                 mysql_dump_command:
                     type: string
                     description: |
@@ -1907,6 +1947,8 @@ properties:
     zabbix:
         type: object
         additionalProperties: false
+        required:
+            - server
         properties:
             itemid:
                 type: integer
@@ -1929,7 +1971,8 @@ properties:
             server:
                 type: string
                 description: |
-                    The address of your Zabbix instance.
+                    The API endpoint URL of your Zabbix instance, usually ending
+                    with "/api_jsonrpc.php". Required.
                 example: https://zabbix.your-domain.com
             username:
                 type: string
@@ -2200,6 +2243,12 @@ properties:
                     - start
                     - finish
                     - fail
+            verify_tls:
+                type: boolean
+                description: |
+                    Verify the TLS certificate of the push URL host. Defaults to
+                    true.
+                example: false
         description: |
             Configuration for a monitoring integration with Uptime Kuma using
             the Push monitor type.
@@ -2402,3 +2451,25 @@ properties:
         description: |
             Configuration for integration with Linux LVM (Logical Volume
             Manager).
+    container:
+        type: object
+        additionalProperties: false
+        properties:
+            secrets_directory:
+                type: string
+                description: |
+                    Secrets directory to use instead of "/run/secrets".
+                example: /path/to/secrets
+        description: |
+            Configuration for integration with Docker or Podman secrets.
+    keepassxc:
+        type: object
+        additionalProperties: false
+        properties:
+            keepassxc_cli_command:
+                type: string
+                description: |
+                    Command to use instead of "keepassxc-cli".
+                example: /usr/local/bin/keepassxc-cli
+        description: |
+            Configuration for integration with the KeePassXC password manager.

borgmatic/execute.py

@@ -1,7 +1,6 @@
 import collections
 import enum
 import logging
-import os
 import select
 import subprocess
 import textwrap
@@ -243,6 +242,9 @@ def mask_command_secrets(full_command):
 MAX_LOGGED_COMMAND_LENGTH = 1000
 
 
+PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG = ('BORG_', 'PG', 'MARIADB_', 'MYSQL_')
+
+
 def log_command(full_command, input_file=None, output_file=None, environment=None):
     '''
     Log the given command (a sequence of command/argument strings), along with its input/output file
@@ -251,14 +253,21 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non
     logger.debug(
         textwrap.shorten(
             ' '.join(
-                tuple(f'{key}=***' for key in (environment or {}).keys())
+                tuple(
+                    f'{key}=***'
+                    for key in (environment or {}).keys()
+                    if any(
+                        key.startswith(prefix)
+                        for prefix in PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG
+                    )
+                )
                 + mask_command_secrets(full_command)
             ),
             width=MAX_LOGGED_COMMAND_LENGTH,
             placeholder=' ...',
         )
-        + (f" < {getattr(input_file, 'name', '')}" if input_file else '')
-        + (f" > {getattr(output_file, 'name', '')}" if output_file else '')
+        + (f" < {getattr(input_file, 'name', input_file)}" if input_file else '')
+        + (f" > {getattr(output_file, 'name', output_file)}" if output_file else '')
     )
 
 
@@ -274,7 +283,7 @@ def execute_command(
     output_file=None,
     input_file=None,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
@@ -284,18 +293,17 @@ def execute_command(
     Execute the given command (a sequence of command/argument strings) and log its output at the
     given log level. If an open output file object is given, then write stdout to the file and only
     log stderr. If an open input file object is given, then read stdin from the file. If shell is
-    True, execute the command within a shell. If an extra environment dict is given, then use it to
-    augment the current environment, and pass the result into the command. If a working directory is
-    given, use that as the present working directory when running the command. If a Borg local path
-    is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning
-    instead of an error. But if Borg exit codes are given as a sequence of exit code configuration
-    dicts, then use that configuration to decide what's an error and what's a warning. If run to
-    completion is False, then return the process for the command without executing it to completion.
+    True, execute the command within a shell. If an environment variables dict is given, then pass
+    it into the command. If a working directory is given, use that as the present working directory
+    when running the command. If a Borg local path is given, and the command matches it (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are
+    given as a sequence of exit code configuration dicts, then use that configuration to decide
+    what's an error and what's a warning. If run to completion is False, then return the process for
+    the command without executing it to completion.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    log_command(full_command, input_file, output_file, extra_environment)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, output_file, environment)
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
@@ -307,8 +315,8 @@ def execute_command(
         shell=shell,
         env=environment,
         cwd=working_directory,
-        # Necessary for the passcommand credential hook to work.
-        close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
+        # Necessary for passing credentials via anonymous pipe.
+        close_fds=False,
     )
     if not run_to_completion:
         return process
@@ -325,39 +333,40 @@ def execute_command(
 
 def execute_command_and_capture_output(
     full_command,
+    input_file=None,
     capture_stderr=False,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its
-    output (stdout). If capture stderr is True, then capture and return stderr in addition to
-    stdout. If shell is True, execute the command within a shell. If an extra environment dict is
-    given, then use it to augment the current environment, and pass the result into the command. If
-    a working directory is given, use that as the present working directory when running the
-    command. If a Borg local path is given, and the command matches it (regardless of arguments),
-    treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a
-    sequence of exit code configuration dicts, then use that configuration to decide what's an error
-    and what's a warning.
+    output (stdout). If an input file descriptor is given, then pipe it to the command's stdin. If
+    capture stderr is True, then capture and return stderr in addition to stdout. If shell is True,
+    execute the command within a shell. If an environment variables dict is given, then pass it into
+    the command. If a working directory is given, use that as the present working directory when
+    running the command. If a Borg local path is given, and the command matches it (regardless of
+    arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are given
+    as a sequence of exit code configuration dicts, then use that configuration to decide what's an
+    error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    log_command(full_command, environment=extra_environment)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, environment=environment)
     command = ' '.join(full_command) if shell else full_command
 
     try:
         output = subprocess.check_output(
             command,
+            stdin=input_file,
             stderr=subprocess.STDOUT if capture_stderr else None,
             shell=shell,
             env=environment,
             cwd=working_directory,
-            # Necessary for the passcommand credential hook to work.
-            close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
+            # Necessary for passing credentials via anonymous pipe.
+            close_fds=False,
         )
     except subprocess.CalledProcessError as error:
         if (
@@ -377,7 +386,7 @@ def execute_command_with_processes(
     output_file=None,
     input_file=None,
     shell=False,
-    extra_environment=None,
+    environment=None,
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
@@ -391,19 +400,17 @@ def execute_command_with_processes(
     If an open output file object is given, then write stdout to the file and only log stderr. But
     if output log level is None, instead suppress logging and return the captured output for (only)
     the given command. If an open input file object is given, then read stdin from the file. If
-    shell is True, execute the command within a shell. If an extra environment dict is given, then
-    use it to augment the current environment, and pass the result into the command. If a working
-    directory is given, use that as the present working directory when running the command. If a
-    Borg local path is given, then for any matching command or process (regardless of arguments),
-    treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a
-    sequence of exit code configuration dicts, then use that configuration to decide what's an error
-    and what's a warning.
+    shell is True, execute the command within a shell. If an environment variables dict is given,
+    then pass it into the command. If a working directory is given, use that as the present working
+    directory when running the command. If a Borg local path is given, then for any matching command
+    or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if
+    Borg exit codes are given as a sequence of exit code configuration dicts, then use that
+    configuration to decide what's an error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
     upstream process.
     '''
-    log_command(full_command, input_file, output_file, extra_environment)
-    environment = {**os.environ, **extra_environment} if extra_environment else None
+    log_command(full_command, input_file, output_file, environment)
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
@@ -418,8 +425,8 @@ def execute_command_with_processes(
             shell=shell,
             env=environment,
             cwd=working_directory,
-            # Necessary for the passcommand credential hook to work.
-            close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
+            # Necessary for passing credentials via anonymous pipe.
+            close_fds=False,
         )
     except (subprocess.CalledProcessError, OSError):
         # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
@@ -430,13 +437,14 @@ def execute_command_with_processes(
                 process.kill()
         raise
 
-    captured_outputs = log_outputs(
-        tuple(processes) + (command_process,),
-        (input_file, output_file),
-        output_log_level,
-        borg_local_path,
-        borg_exit_codes,
-    )
+    with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
+        captured_outputs = log_outputs(
+            tuple(processes) + (command_process,),
+            (input_file, output_file),
+            output_log_level,
+            borg_local_path,
+            borg_exit_codes,
+        )
 
     if output_log_level is None:
         return captured_outputs.get(command_process)

borgmatic/hooks/command.py

@@ -30,16 +30,18 @@ def interpolate_context(hook_description, command, context):
 
 def make_environment(current_environment, sys_module=sys):
     '''
-    Given the existing system environment as a map from environment variable name to value, return
-    (in the same form) any extra environment variables that should be used when running command
-    hooks.
+    Given the existing system environment as a map from environment variable name to value, return a
+    copy of it, augmented with any extra environment variables that should be used when running
+    command hooks.
     '''
+    environment = dict(current_environment)
+
     # Detect whether we're running within a PyInstaller bundle. If so, set or clear LD_LIBRARY_PATH
     # based on the value of LD_LIBRARY_PATH_ORIG. This prevents library version information errors.
     if getattr(sys_module, 'frozen', False) and hasattr(sys_module, '_MEIPASS'):
-        return {'LD_LIBRARY_PATH': current_environment.get('LD_LIBRARY_PATH_ORIG', '')}
+        environment['LD_LIBRARY_PATH'] = environment.get('LD_LIBRARY_PATH_ORIG', '')
 
-    return {}
+    return environment
 
 
 def execute_hook(commands, umask, config_filename, description, dry_run, **context):
@@ -85,7 +87,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
                 [command],
                 output_log_level=(logging.ERROR if description == 'on-error' else logging.WARNING),
                 shell=True,
-                extra_environment=make_environment(os.environ),
+                environment=make_environment(os.environ),
             )
     finally:
         if original_umask:

borgmatic/hooks/credential/container.py

@@ -0,0 +1,43 @@
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+SECRET_NAME_PATTERN = re.compile(r'^\w+$')
+DEFAULT_SECRETS_DIRECTORY = '/run/secrets'
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a secret name to load, read the secret from the corresponding container secrets file
+    and return it.
+
+    Raise ValueError if the credential parameters is not one element, the secret name is invalid, or
+    the secret file cannot be read.
+    '''
+    try:
+        (secret_name,) = credential_parameters
+    except ValueError:
+        name = ' '.join(credential_parameters)
+
+        raise ValueError(f'Cannot load invalid secret name: "{name}"')
+
+    if not SECRET_NAME_PATTERN.match(secret_name):
+        raise ValueError(f'Cannot load invalid secret name: "{secret_name}"')
+
+    try:
+        with open(
+            os.path.join(
+                config.get('working_directory', ''),
+                (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY),
+                secret_name,
+            )
+        ) as secret_file:
+            return secret_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load secret "{secret_name}" from file: {error.filename}')

borgmatic/hooks/credential/file.py

@@ -0,0 +1,30 @@
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a credential path to load, load the credential from file and return it.
+
+    Raise ValueError if the credential parameters is not one element or the secret file cannot be
+    read.
+    '''
+    try:
+        (credential_path,) = credential_parameters
+    except ValueError:
+        name = ' '.join(credential_parameters)
+
+        raise ValueError(f'Cannot load invalid credential: "{name}"')
+
+    try:
+        with open(
+            os.path.join(config.get('working_directory', ''), credential_path)
+        ) as credential_file:
+            return credential_file.read().rstrip(os.linesep)
+    except (FileNotFoundError, OSError) as error:
+        logger.warning(error)
+
+        raise ValueError(f'Cannot load credential file: {error.filename}')

borgmatic/hooks/credential/keepassxc.py

@@ -0,0 +1,42 @@
+import logging
+import os
+import shlex
+
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def load_credential(hook_config, config, credential_parameters):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a KeePassXC database path and an attribute name to load, run keepassxc-cli to fetch
+    the corresponidng KeePassXC credential and return it.
+
+    Raise ValueError if keepassxc-cli can't retrieve the credential.
+    '''
+    try:
+        (database_path, attribute_name) = credential_parameters
+    except ValueError:
+        path_and_name = ' '.join(credential_parameters)
+
+        raise ValueError(
+            f'Cannot load credential with invalid KeePassXC database path and attribute name: "{path_and_name}"'
+        )
+
+    if not os.path.exists(database_path):
+        raise ValueError(
+            f'Cannot load credential because KeePassXC database path does not exist: {database_path}'
+        )
+
+    return borgmatic.execute.execute_command_and_capture_output(
+        tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli')))
+        + (
+            'show',
+            '--show-protected',
+            '--attributes',
+            'Password',
+            database_path,
+            attribute_name,
+        )
+    ).rstrip(os.linesep)

borgmatic/hooks/credential/parse.py

@@ -1,42 +1,124 @@
 import functools
 import re
+import shlex
 
 import borgmatic.hooks.dispatch
 
 IS_A_HOOK = False
 
 
-CREDENTIAL_PATTERN = re.compile(
-    r'\{credential +(?P<hook_name>[A-Za-z0-9_]+) +(?P<credential_name>[A-Za-z0-9_]+)\}'
-)
-
-GENERAL_CREDENTIAL_PATTERN = re.compile(r'\{credential( +[^}]*)?\}')
-
-
-@functools.cache
-def resolve_credential(value):
+class Hash_adapter:
     '''
-    Given a configuration value containing a string like "{credential hookname credentialname}", resolve it by
-    calling the relevant hook to get the actual credential value. If the given value does not
-    actually contain a credential tag, then return it unchanged.
+    A Hash_adapter instance wraps an unhashable object and pretends it's hashable. This is intended
+    for passing to a @functools.cache-decorated function to prevent it from complaining that an
+    argument is unhashable. It should only be used for arguments that you don't want to actually
+    impact the cache hashing, because Hash_adapter doesn't actually hash the object's contents.
 
-    Cache the value so repeated calls to this function don't need to load the credential repeatedly.
+    Example usage:
+
+        @functools.cache
+        def func(a, b):
+            print(a, b.actual_value)
+            return a
+
+        func(5, Hash_adapter({1: 2, 3: 4}))  # Calls func(), prints, and returns.
+        func(5, Hash_adapter({1: 2, 3: 4}))  # Hits the cache and just returns the value.
+        func(5, Hash_adapter({5: 6, 7: 8}))  # Also uses cache, since the Hash_adapter is ignored.
+
+    In the above function, the "b" value is one that has been wrapped with Hash_adappter, and
+    therefore "b.actual_value" is necessary to access the original value.
+    '''
+
+    def __init__(self, actual_value):
+        self.actual_value = actual_value
+
+    def __eq__(self, other):
+        return True
+
+    def __hash__(self):
+        return 0
+
+
+UNHASHABLE_TYPES = (dict, list, set)
+
+
+def cache_ignoring_unhashable_arguments(function):
+    '''
+    A function decorator that caches calls to the decorated function but ignores any unhashable
+    arguments when performing cache lookups. This is intended to be a drop-in replacement for
+    functools.cache.
+
+    Example usage:
+
+        @cache_ignoring_unhashable_arguments
+        def func(a, b):
+            print(a, b)
+            return a
+
+        func(5, {1: 2, 3: 4})  # Calls func(), prints, and returns.
+        func(5, {1: 2, 3: 4})  # Hits the cache and just returns the value.
+        func(5, {5: 6, 7: 8})  # Also uses cache, since the unhashable value (the dict) is ignored.
+    '''
+
+    @functools.cache
+    def cached_function(*args, **kwargs):
+        return function(
+            *(arg.actual_value if isinstance(arg, Hash_adapter) else arg for arg in args),
+            **{
+                key: value.actual_value if isinstance(value, Hash_adapter) else value
+                for (key, value) in kwargs.items()
+            },
+        )
+
+    @functools.wraps(function)
+    def wrapper_function(*args, **kwargs):
+        return cached_function(
+            *(Hash_adapter(arg) if isinstance(arg, UNHASHABLE_TYPES) else arg for arg in args),
+            **{
+                key: Hash_adapter(value) if isinstance(value, UNHASHABLE_TYPES) else value
+                for (key, value) in kwargs.items()
+            },
+        )
+
+    wrapper_function.cache_clear = cached_function.cache_clear
+
+    return wrapper_function
+
+
+CREDENTIAL_PATTERN = re.compile(r'\{credential( +(?P<hook_and_parameters>.*))?\}')
+
+
+@cache_ignoring_unhashable_arguments
+def resolve_credential(value, config):
+    '''
+    Given a configuration value containing a string like "{credential hookname credentialname}" and
+    a configuration dict, resolve the credential by calling the relevant hook to get the actual
+    credential value. If the given value does not actually contain a credential tag, then return it
+    unchanged.
+
+    Cache the value (ignoring the config for purposes of caching), so repeated calls to this
+    function don't need to load the credential repeatedly.
 
     Raise ValueError if the config could not be parsed or the credential could not be loaded.
     '''
     if value is None:
         return value
 
-    result = CREDENTIAL_PATTERN.sub(
-        lambda matcher: borgmatic.hooks.dispatch.call_hook(
-            'load_credential', {}, matcher.group('hook_name'), matcher.group('credential_name')
-        ),
-        value,
-    )
+    matcher = CREDENTIAL_PATTERN.match(value)
 
-    # If we've tried to parse the credential, but the parsed result still looks kind of like a
-    # credential, it means it's invalid syntax.
-    if GENERAL_CREDENTIAL_PATTERN.match(result):
+    if not matcher:
+        return value
+
+    hook_and_parameters = matcher.group('hook_and_parameters')
+
+    if not hook_and_parameters:
         raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
 
-    return result
+    (hook_name, *credential_parameters) = shlex.split(hook_and_parameters)
+
+    if not credential_parameters:
+        raise ValueError(f'Cannot load credential with invalid syntax "{value}"')
+
+    return borgmatic.hooks.dispatch.call_hook(
+        'load_credential', config, hook_name, tuple(credential_parameters)
+    )

borgmatic/hooks/credential/systemd.py

@@ -8,14 +8,22 @@ logger = logging.getLogger(__name__)
 CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$')
 
 
-def load_credential(hook_config, config, credential_name):
+def load_credential(hook_config, config, credential_parameters):
     '''
-    Given the hook configuration dict, the configuration dict, and a credential name to load, read
-    the credential from the corresponding systemd credential file and return it.
+    Given the hook configuration dict, the configuration dict, and a credential parameters tuple
+    containing a credential name to load, read the credential from the corresponding systemd
+    credential file and return it.
 
     Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the
     credential name is invalid, or the credential file cannot be read.
     '''
+    try:
+        (credential_name,) = credential_parameters
+    except ValueError:
+        name = ' '.join(credential_parameters)
+
+        raise ValueError(f'Cannot load invalid credential name: "{name}"')
+
     credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
 
     if not credentials_directory:

borgmatic/hooks/data_source/bootstrap.py

@@ -55,9 +55,17 @@ def dump_data_sources(
             manifest_file,
         )
 
-    patterns.extend(borgmatic.borg.pattern.Pattern(config_path) for config_path in config_paths)
+    patterns.extend(
+        borgmatic.borg.pattern.Pattern(
+            config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK
+        )
+        for config_path in config_paths
+    )
     patterns.append(
-        borgmatic.borg.pattern.Pattern(os.path.join(borgmatic_runtime_directory, 'bootstrap'))
+        borgmatic.borg.pattern.Pattern(
+            os.path.join(borgmatic_runtime_directory, 'bootstrap'),
+            source=borgmatic.borg.pattern.Pattern_source.HOOK,
+        )
     )
 
     return []

borgmatic/hooks/data_source/btrfs.py

@@ -54,7 +54,9 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
     between the current Btrfs filesystem and subvolume mount points and the paths of any patterns.
     The idea is that these pattern paths represent the requested subvolumes to snapshot.
 
-    If patterns is None, then return all subvolumes, sorted by path.
+    Only include subvolumes that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then
+    return all subvolumes instead, sorted by path.
 
     Return the result as a sequence of matching subvolume mount points.
     '''
@@ -73,7 +75,12 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
                     mount_point, candidate_patterns
                 ),
             )
-            if patterns is None or contained_patterns
+            if patterns is None
+            or any(
+                pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                for pattern in contained_patterns
+            )
         )
 
     return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path))
@@ -121,6 +128,7 @@ def make_snapshot_exclude_pattern(subvolume_path):  # pragma: no cover
         ),
         borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
         borgmatic.borg.pattern.Pattern_style.FNMATCH,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -153,6 +161,7 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
         pattern.type,
         pattern.style,
         pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -198,7 +207,8 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}')
 
-    # Based on the configured patterns, determine Btrfs subvolumes to backup.
+    # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those
+    # patterns that came from actual user configuration (as opposed to, say, other hooks).
     btrfs_command = hook_config.get('btrfs_command', 'btrfs')
     findmnt_command = hook_config.get('findmnt_command', 'findmnt')
     subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns)
@@ -299,9 +309,12 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                 logger.debug(error)
                 return
 
-            # Strip off the subvolume path from the end of the snapshot path and then delete the
-            # resulting directory.
-            shutil.rmtree(snapshot_path.rsplit(subvolume.path, 1)[0])
+            # Remove the snapshot parent directory if it still exists. (It might not exist if the
+            # snapshot was for "/".)
+            snapshot_parent_dir = snapshot_path.rsplit(subvolume.path, 1)[0]
+
+            if os.path.isdir(snapshot_parent_dir):
+                shutil.rmtree(snapshot_parent_dir)
 
 
 def make_data_source_dump_patterns(

borgmatic/hooks/data_source/lvm.py

@@ -1,5 +1,6 @@
 import collections
 import glob
+import hashlib
 import json
 import logging
 import os
@@ -33,7 +34,9 @@ def get_logical_volumes(lsblk_command, patterns=None):
     between the current LVM logical volume mount points and the paths of any patterns. The idea is
     that these pattern paths represent the requested logical volumes to snapshot.
 
-    If patterns is None, include all logical volume mounts points, not just those in patterns.
+    Only include logical volumes that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, include
+    all logical volume mounts points instead, not just those in patterns.
 
     Return the result as a sequence of Logical_volume instances.
     '''
@@ -72,7 +75,12 @@ def get_logical_volumes(lsblk_command, patterns=None):
                     device['mountpoint'], candidate_patterns
                 ),
             )
-            if not patterns or contained_patterns
+            if not patterns
+            or any(
+                pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                for pattern in contained_patterns
+            )
         )
     except KeyError as error:
         raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"')
@@ -124,10 +132,14 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # prag
     )
 
 
-def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
+MOUNT_POINT_HASH_LENGTH = 10
+
+
+def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory):
     '''
-    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
-    path rewritten to be in a snapshot directory based on the given runtime directory.
+    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume
+    containing it, return a new Pattern with its path rewritten to be in a snapshot directory based
+    on both the given runtime directory and the given Logical_volume's mount point.
 
     Move any initial caret in a regular expression pattern path to the beginning, so as not to break
     the regular expression.
@@ -142,6 +154,13 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
     rewritten_path = initial_caret + os.path.join(
         normalized_runtime_directory,
         'lvm_snapshots',
+        # Including this hash prevents conflicts between snapshot patterns for different logical
+        # volumes. For instance, without this, snapshotting a logical volume at /var and another at
+        # /var/spool would result in overlapping snapshot patterns and therefore colliding mount
+        # attempts.
+        hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
+            MOUNT_POINT_HASH_LENGTH
+        ),
         '.',  # Borg 1.4+ "slashdot" hack.
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
@@ -152,6 +171,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
         pattern.type,
         pattern.style,
         pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -180,7 +200,8 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'Snapshotting LVM logical volumes{dry_run_label}')
 
-    # List logical volumes to get their mount points.
+    # List logical volumes to get their mount points, but only consider those patterns that came
+    # from actual user configuration (as opposed to, say, other hooks).
     lsblk_command = hook_config.get('lsblk_command', 'lsblk')
     requested_logical_volumes = get_logical_volumes(lsblk_command, patterns)
 
@@ -218,6 +239,9 @@ def dump_data_sources(
         snapshot_mount_path = os.path.join(
             normalized_runtime_directory,
             'lvm_snapshots',
+            hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
+                MOUNT_POINT_HASH_LENGTH
+            ),
             logical_volume.mount_point.lstrip(os.path.sep),
         )
 
@@ -233,7 +257,9 @@ def dump_data_sources(
         )
 
         for pattern in logical_volume.contained_patterns:
-            snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
+            snapshot_pattern = make_borg_snapshot_pattern(
+                pattern, logical_volume, normalized_runtime_directory
+            )
 
             # Attempt to update the pattern in place, since pattern order matters to Borg.
             try:
@@ -337,6 +363,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
             os.path.normpath(borgmatic_runtime_directory),
         ),
         'lvm_snapshots',
+        '*',
     )
     logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
     umount_command = hook_config.get('umount_command', 'umount')
@@ -349,7 +376,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
             snapshot_mount_path = os.path.join(
                 snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep)
             )
-            if not os.path.isdir(snapshot_mount_path):
+
+            # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
+            # logical volume and therefore there's nothing to unmount.
+            if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
                 continue
 
             # This might fail if the directory is already mounted, but we swallow errors here since
@@ -374,7 +404,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                 return
             except subprocess.CalledProcessError as error:
                 logger.debug(error)
-                return
+                continue
 
         if not dry_run:
             shutil.rmtree(snapshots_directory)

borgmatic/hooks/data_source/mariadb.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 import os
+import re
 import shlex
 
 import borgmatic.borg.pattern
@@ -23,14 +24,90 @@ def make_dump_path(base_directory):  # pragma: no cover
     return dump.make_data_source_dump_path(base_directory, 'mariadb_databases')
 
 
-SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
+DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile('^--defaults-extra-file=(?P<filename>.*)$')
 
 
-def database_names_to_dump(database, extra_environment, dry_run):
+def parse_extra_options(extra_options):
     '''
-    Given a requested database config, return the corresponding sequence of database names to dump.
-    In the case of "all", query for the names of databases on the configured host and return them,
-    excluding any system databases that will cause problems during restore.
+    Given an extra options string, split the options into a tuple and return it. Additionally, if
+    the first option is "--defaults-extra-file=...", then remove it from the options and return the
+    filename.
+
+    So the return value is a tuple of: (parsed options, defaults extra filename).
+
+    The intent is to support downstream merging of multiple "--defaults-extra-file"s, as
+    MariaDB/MySQL only allows one at a time.
+    '''
+    split_extra_options = tuple(shlex.split(extra_options)) if extra_options else ()
+
+    if not split_extra_options:
+        return ((), None)
+
+    match = DEFAULTS_EXTRA_FILE_FLAG_PATTERN.match(split_extra_options[0])
+
+    if not match:
+        return (split_extra_options, None)
+
+    return (split_extra_options[1:], match.group('filename'))
+
+
+def make_defaults_file_options(username=None, password=None, defaults_extra_filename=None):
+    '''
+    Given a database username and/or password, write it to an anonymous pipe and return the flags
+    for passing that file descriptor to an executed command. The idea is that this is a more secure
+    way to transmit credentials to a database client than using an environment variable.
+
+    If no username or password are given, then return the options for the given defaults extra
+    filename (if any). But if there is a username and/or password and a defaults extra filename is
+    given, then "!include" it from the generated file, effectively allowing multiple defaults extra
+    files.
+
+    Do not use the returned value for multiple different command invocations. That will not work
+    because each pipe is "used up" once read.
+    '''
+    values = '\n'.join(
+        (
+            (f'user={username}' if username is not None else ''),
+            (f'password={password}' if password is not None else ''),
+        )
+    ).strip()
+
+    if not values:
+        if defaults_extra_filename:
+            return (f'--defaults-extra-file={defaults_extra_filename}',)
+
+        return ()
+
+    fields_message = ' and '.join(
+        field_name
+        for field_name in (
+            (f'username ({username})' if username is not None else None),
+            ('password' if password is not None else None),
+        )
+        if field_name is not None
+    )
+    include_message = f' (including {defaults_extra_filename})' if defaults_extra_filename else ''
+    logger.debug(f'Writing database {fields_message} to defaults extra file pipe{include_message}')
+
+    include = f'!include {defaults_extra_filename}\n' if defaults_extra_filename else ''
+
+    read_file_descriptor, write_file_descriptor = os.pipe()
+    os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode('utf-8'))
+    os.close(write_file_descriptor)
+
+    # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database
+    # client child process to inherit the file descriptor.
+    os.set_inheritable(read_file_descriptor, True)
+
+    return (f'--defaults-extra-file=/dev/fd/{read_file_descriptor}',)
+
+
+def database_names_to_dump(database, config, username, password, environment, dry_run):
+    '''
+    Given a requested database config, a configuration dict, a database username and password, an
+    environment dict, and whether this is a dry run, return the corresponding sequence of database
+    names to dump. In the case of "all", query for the names of databases on the configured host and
+    return them, excluding any system databases that will cause problems during restore.
     '''
     if database['name'] != 'all':
         return (database['name'],)
@@ -40,24 +117,23 @@ def database_names_to_dump(database, extra_environment, dry_run):
     mariadb_show_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb')
     )
+    extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options'))
     show_command = (
         mariadb_show_command
-        + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
+        + make_defaults_file_options(username, password, defaults_extra_filename)
+        + extra_options
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (
-            ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username']))
-            if 'username' in database
-            else ()
-        )
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
+
     logger.debug('Querying for "all" MariaDB databases to dump')
-    show_output = execute_command_and_capture_output(
-        show_command, extra_environment=extra_environment
-    )
+
+    show_output = execute_command_and_capture_output(show_command, environment=environment)
 
     return tuple(
         show_name
@@ -66,8 +142,19 @@ def database_names_to_dump(database, extra_environment, dry_run):
     )
 
 
+SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
+
+
 def execute_dump_command(
-    database, dump_path, database_names, extra_environment, dry_run, dry_run_label
+    database,
+    config,
+    username,
+    password,
+    dump_path,
+    database_names,
+    environment,
+    dry_run,
+    dry_run_label,
 ):
     '''
     Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
@@ -94,18 +181,17 @@ def execute_dump_command(
         shlex.quote(part)
         for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump')
     )
+    extra_options, defaults_extra_filename = parse_extra_options(database.get('options'))
     dump_command = (
         mariadb_dump_command
-        + (tuple(database['options'].split(' ')) if 'options' in database else ())
+        + make_defaults_file_options(username, password, defaults_extra_filename)
+        + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (
-            ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username']))
-            if 'username' in database
-            else ()
-        )
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -119,7 +205,7 @@ def execute_dump_command(
 
     return execute_command(
         dump_command,
-        extra_environment=extra_environment,
+        environment=environment,
         run_to_completion=False,
     )
 
@@ -161,12 +247,16 @@ def dump_data_sources(
 
     for database in databases:
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        extra_environment = (
-            {'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(database['password'])}
-            if 'password' in database
-            else None
+        username = borgmatic.hooks.credential.parse.resolve_credential(
+            database.get('username'), config
+        )
+        password = borgmatic.hooks.credential.parse.resolve_credential(
+            database.get('password'), config
+        )
+        environment = dict(os.environ)
+        dump_database_names = database_names_to_dump(
+            database, config, username, password, environment, dry_run
         )
-        dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
 
         if not dump_database_names:
             if dry_run:
@@ -181,9 +271,12 @@ def dump_data_sources(
                 processes.append(
                     execute_dump_command(
                         renamed_database,
+                        config,
+                        username,
+                        password,
                         dump_path,
                         (dump_name,),
-                        extra_environment,
+                        environment,
                         dry_run,
                         dry_run_label,
                     )
@@ -192,9 +285,12 @@ def dump_data_sources(
             processes.append(
                 execute_dump_command(
                     database,
+                    config,
+                    username,
+                    password,
                     dump_path,
                     dump_database_names,
-                    extra_environment,
+                    environment,
                     dry_run,
                     dry_run_label,
                 )
@@ -203,7 +299,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'mariadb_databases')
+                os.path.join(borgmatic_runtime_directory, 'mariadb_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
@@ -264,32 +361,38 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
+    tls = data_source.get('restore_tls', data_source.get('tls'))
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        connection_params['username']
-        or data_source.get('restore_username', data_source.get('username'))
+        (
+            connection_params['username']
+            or data_source.get('restore_username', data_source.get('username'))
+        ),
+        config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        connection_params['password']
-        or data_source.get('restore_password', data_source.get('password'))
+        (
+            connection_params['password']
+            or data_source.get('restore_password', data_source.get('password'))
+        ),
+        config,
     )
 
     mariadb_restore_command = tuple(
         shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb')
     )
+    extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options'))
     restore_command = (
         mariadb_restore_command
+        + make_defaults_file_options(username, password, defaults_extra_filename)
+        + extra_options
         + ('--batch',)
-        + (
-            tuple(data_source['restore_options'].split(' '))
-            if 'restore_options' in data_source
-            else ()
-        )
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
-        + (('--user', username) if username else ())
+        + (('--ssl',) if tls is True else ())
+        + (('--skip-ssl',) if tls is False else ())
     )
-    extra_environment = {'MYSQL_PWD': password} if password else None
+    environment = dict(os.environ)
 
     logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}")
     if dry_run:
@@ -302,5 +405,5 @@ def restore_data_source_dump(
         [extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment=extra_environment,
+        environment=environment,
     )

borgmatic/hooks/data_source/mongodb.py

@@ -69,7 +69,7 @@ def dump_data_sources(
         if dry_run:
             continue
 
-        command = build_dump_command(database, dump_filename, dump_format)
+        command = build_dump_command(database, config, dump_filename, dump_format)
 
         if dump_format == 'directory':
             dump.create_parent_directory_for_dump(dump_filename)
@@ -81,19 +81,44 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'mongodb_databases')
+                os.path.join(borgmatic_runtime_directory, 'mongodb_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
     return processes
 
 
-def build_dump_command(database, dump_filename, dump_format):
+def make_password_config_file(password):
+    '''
+    Given a database password, write it as a MongoDB configuration file to an anonymous pipe and
+    return its filename. The idea is that this is a more secure way to transmit a password to
+    MongoDB than providing it directly on the command-line.
+
+    Do not use the returned value for multiple different command invocations. That will not work
+    because each pipe is "used up" once read.
+    '''
+    logger.debug('Writing MongoDB password to configuration file pipe')
+
+    read_file_descriptor, write_file_descriptor = os.pipe()
+    os.write(write_file_descriptor, f'password: {password}'.encode('utf-8'))
+    os.close(write_file_descriptor)
+
+    # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database
+    # client child process to inherit the file descriptor.
+    os.set_inheritable(read_file_descriptor, True)
+
+    return f'/dev/fd/{read_file_descriptor}'
+
+
+def build_dump_command(database, config, dump_filename, dump_format):
     '''
     Return the mongodump command from a single database configuration.
     '''
     all_databases = database['name'] == 'all'
 
+    password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config)
+
     return (
         ('mongodump',)
         + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
@@ -103,22 +128,15 @@ def build_dump_command(database, dump_filename, dump_format):
             (
                 '--username',
                 shlex.quote(
-                    borgmatic.hooks.credential.parse.resolve_credential(database['username'])
+                    borgmatic.hooks.credential.parse.resolve_credential(
+                        database['username'], config
+                    )
                 ),
             )
             if 'username' in database
             else ()
         )
-        + (
-            (
-                '--password',
-                shlex.quote(
-                    borgmatic.hooks.credential.parse.resolve_credential(database['password'])
-                ),
-            )
-            if 'password' in database
-            else ()
-        )
+        + (('--config', make_password_config_file(password)) if password else ())
         + (
             ('--authenticationDatabase', shlex.quote(database['authentication_database']))
             if 'authentication_database' in database
@@ -192,7 +210,7 @@ def restore_data_source_dump(
         data_source.get('hostname'),
     )
     restore_command = build_restore_command(
-        extract_process, data_source, dump_filename, connection_params
+        extract_process, data_source, config, dump_filename, connection_params
     )
 
     logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}")
@@ -209,7 +227,7 @@ def restore_data_source_dump(
     )
 
 
-def build_restore_command(extract_process, database, dump_filename, connection_params):
+def build_restore_command(extract_process, database, config, dump_filename, connection_params):
     '''
     Return the mongorestore command from a single database configuration.
     '''
@@ -218,10 +236,18 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
     )
     port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        connection_params['username'] or database.get('restore_username', database.get('username'))
+        (
+            connection_params['username']
+            or database.get('restore_username', database.get('username'))
+        ),
+        config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        connection_params['password'] or database.get('restore_password', database.get('password'))
+        (
+            connection_params['password']
+            or database.get('restore_password', database.get('password'))
+        ),
+        config,
     )
 
     command = ['mongorestore']
@@ -238,7 +264,7 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
     if username:
         command.extend(('--username', username))
     if password:
-        command.extend(('--password', password))
+        command.extend(('--config', make_password_config_file(password)))
     if 'authentication_database' in database:
         command.extend(('--authenticationDatabase', database['authentication_database']))
     if 'restore_options' in database:

borgmatic/hooks/data_source/mysql.py

@@ -6,6 +6,7 @@ import shlex
 import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
+import borgmatic.hooks.data_source.mariadb
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
@@ -26,11 +27,12 @@ def make_dump_path(base_directory):  # pragma: no cover
 SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
 
 
-def database_names_to_dump(database, extra_environment, dry_run):
+def database_names_to_dump(database, config, username, password, environment, dry_run):
     '''
-    Given a requested database config, return the corresponding sequence of database names to dump.
-    In the case of "all", query for the names of databases on the configured host and return them,
-    excluding any system databases that will cause problems during restore.
+    Given a requested database config, a configuration dict, a database username and password, an
+    environment dict, and whether this is a dry run, return the corresponding sequence of database
+    names to dump. In the case of "all", query for the names of databases on the configured host and
+    return them, excluding any system databases that will cause problems during restore.
     '''
     if database['name'] != 'all':
         return (database['name'],)
@@ -40,24 +42,27 @@ def database_names_to_dump(database, extra_environment, dry_run):
     mysql_show_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql')
     )
+    extra_options, defaults_extra_filename = (
+        borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options'))
+    )
     show_command = (
         mysql_show_command
-        + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
+        + borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+            username, password, defaults_extra_filename
+        )
+        + extra_options
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (
-            ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username']))
-            if 'username' in database
-            else ()
-        )
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
+
     logger.debug('Querying for "all" MySQL databases to dump')
-    show_output = execute_command_and_capture_output(
-        show_command, extra_environment=extra_environment
-    )
+
+    show_output = execute_command_and_capture_output(show_command, environment=environment)
 
     return tuple(
         show_name
@@ -67,7 +72,15 @@ def database_names_to_dump(database, extra_environment, dry_run):
 
 
 def execute_dump_command(
-    database, dump_path, database_names, extra_environment, dry_run, dry_run_label
+    database,
+    config,
+    username,
+    password,
+    dump_path,
+    database_names,
+    environment,
+    dry_run,
+    dry_run_label,
 ):
     '''
     Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a
@@ -93,18 +106,21 @@ def execute_dump_command(
     mysql_dump_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump')
     )
+    extra_options, defaults_extra_filename = (
+        borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options'))
+    )
     dump_command = (
         mysql_dump_command
-        + (tuple(database['options'].split(' ')) if 'options' in database else ())
+        + borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+            username, password, defaults_extra_filename
+        )
+        + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (
-            ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username']))
-            if 'username' in database
-            else ()
-        )
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -118,7 +134,7 @@ def execute_dump_command(
 
     return execute_command(
         dump_command,
-        extra_environment=extra_environment,
+        environment=environment,
         run_to_completion=False,
     )
 
@@ -160,12 +176,16 @@ def dump_data_sources(
 
     for database in databases:
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        extra_environment = (
-            {'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(database['password'])}
-            if 'password' in database
-            else None
+        username = borgmatic.hooks.credential.parse.resolve_credential(
+            database.get('username'), config
+        )
+        password = borgmatic.hooks.credential.parse.resolve_credential(
+            database.get('password'), config
+        )
+        environment = dict(os.environ)
+        dump_database_names = database_names_to_dump(
+            database, config, username, password, environment, dry_run
         )
-        dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
 
         if not dump_database_names:
             if dry_run:
@@ -180,9 +200,12 @@ def dump_data_sources(
                 processes.append(
                     execute_dump_command(
                         renamed_database,
+                        config,
+                        username,
+                        password,
                         dump_path,
                         (dump_name,),
-                        extra_environment,
+                        environment,
                         dry_run,
                         dry_run_label,
                     )
@@ -191,9 +214,12 @@ def dump_data_sources(
             processes.append(
                 execute_dump_command(
                     database,
+                    config,
+                    username,
+                    password,
                     dump_path,
                     dump_database_names,
-                    extra_environment,
+                    environment,
                     dry_run,
                     dry_run_label,
                 )
@@ -202,7 +228,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'mysql_databases')
+                os.path.join(borgmatic_runtime_directory, 'mysql_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
@@ -263,32 +290,42 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
+    tls = data_source.get('restore_tls', data_source.get('tls'))
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        connection_params['username']
-        or data_source.get('restore_username', data_source.get('username'))
+        (
+            connection_params['username']
+            or data_source.get('restore_username', data_source.get('username'))
+        ),
+        config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        connection_params['password']
-        or data_source.get('restore_password', data_source.get('password'))
+        (
+            connection_params['password']
+            or data_source.get('restore_password', data_source.get('password'))
+        ),
+        config,
     )
 
     mysql_restore_command = tuple(
         shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql')
     )
+    extra_options, defaults_extra_filename = (
+        borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options'))
+    )
     restore_command = (
         mysql_restore_command
-        + ('--batch',)
-        + (
-            tuple(data_source['restore_options'].split(' '))
-            if 'restore_options' in data_source
-            else ()
+        + borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
+            username, password, defaults_extra_filename
         )
+        + extra_options
+        + ('--batch',)
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
-        + (('--user', username) if username else ())
+        + (('--ssl',) if tls is True else ())
+        + (('--skip-ssl',) if tls is False else ())
     )
-    extra_environment = {'MYSQL_PWD': password} if password else None
+    environment = dict(os.environ)
 
     logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}")
     if dry_run:
@@ -301,5 +338,5 @@ def restore_data_source_dump(
         [extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment=extra_environment,
+        environment=environment,
     )

borgmatic/hooks/data_source/postgresql.py

@@ -25,49 +25,52 @@ def make_dump_path(base_directory):  # pragma: no cover
     return dump.make_data_source_dump_path(base_directory, 'postgresql_databases')
 
 
-def make_extra_environment(database, restore_connection_params=None):
+def make_environment(database, config, restore_connection_params=None):
     '''
-    Make the extra_environment dict from the given database configuration. If restore connection
-    params are given, this is for a restore operation.
+    Make an environment dict from the current environment variables and the given database
+    configuration. If restore connection params are given, this is for a restore operation.
     '''
-    extra = dict()
+    environment = dict(os.environ)
 
     try:
         if restore_connection_params:
-            extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                restore_connection_params.get('password')
-                or database.get('restore_password', database['password'])
+            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
+                (
+                    restore_connection_params.get('password')
+                    or database.get('restore_password', database['password'])
+                ),
+                config,
             )
         else:
-            extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                database['password']
+            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
+                database['password'], config
             )
     except (AttributeError, KeyError):
         pass
 
     if 'ssl_mode' in database:
-        extra['PGSSLMODE'] = database['ssl_mode']
+        environment['PGSSLMODE'] = database['ssl_mode']
     if 'ssl_cert' in database:
-        extra['PGSSLCERT'] = database['ssl_cert']
+        environment['PGSSLCERT'] = database['ssl_cert']
     if 'ssl_key' in database:
-        extra['PGSSLKEY'] = database['ssl_key']
+        environment['PGSSLKEY'] = database['ssl_key']
     if 'ssl_root_cert' in database:
-        extra['PGSSLROOTCERT'] = database['ssl_root_cert']
+        environment['PGSSLROOTCERT'] = database['ssl_root_cert']
     if 'ssl_crl' in database:
-        extra['PGSSLCRL'] = database['ssl_crl']
+        environment['PGSSLCRL'] = database['ssl_crl']
 
-    return extra
+    return environment
 
 
 EXCLUDED_DATABASE_NAMES = ('template0', 'template1')
 
 
-def database_names_to_dump(database, extra_environment, dry_run):
+def database_names_to_dump(database, config, environment, dry_run):
     '''
-    Given a requested database config, return the corresponding sequence of database names to dump.
-    In the case of "all" when a database format is given, query for the names of databases on the
-    configured host and return them. For "all" without a database format, just return a sequence
-    containing "all".
+    Given a requested database config and a configuration dict, return the corresponding sequence of
+    database names to dump. In the case of "all" when a database format is given, query for the
+    names of databases on the configured host and return them. For "all" without a database format,
+    just return a sequence containing "all".
     '''
     requested_name = database['name']
 
@@ -89,7 +92,7 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + (
             (
                 '--username',
-                borgmatic.hooks.credential.parse.resolve_credential(database['username']),
+                borgmatic.hooks.credential.parse.resolve_credential(database['username'], config),
             )
             if 'username' in database
             else ()
@@ -97,9 +100,7 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
     )
     logger.debug('Querying for "all" PostgreSQL databases to dump')
-    list_output = execute_command_and_capture_output(
-        list_command, extra_environment=extra_environment
-    )
+    list_output = execute_command_and_capture_output(list_command, environment=environment)
 
     return tuple(
         row[0]
@@ -146,9 +147,9 @@ def dump_data_sources(
     logger.info(f'Dumping PostgreSQL databases{dry_run_label}')
 
     for database in databases:
-        extra_environment = make_extra_environment(database)
+        environment = make_environment(database, config)
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
+        dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
             if dry_run:
@@ -158,6 +159,7 @@ def dump_data_sources(
 
         for database_name in dump_database_names:
             dump_format = database.get('format', None if database_name == 'all' else 'custom')
+            compression = database.get('compression')
             default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump'
             dump_command = tuple(
                 shlex.quote(part)
@@ -189,7 +191,7 @@ def dump_data_sources(
                         '--username',
                         shlex.quote(
                             borgmatic.hooks.credential.parse.resolve_credential(
-                                database['username']
+                                database['username'], config
                             )
                         ),
                     )
@@ -198,6 +200,7 @@ def dump_data_sources(
                 )
                 + (('--no-owner',) if database.get('no_owner', False) else ())
                 + (('--format', shlex.quote(dump_format)) if dump_format else ())
+                + (('--compress', shlex.quote(str(compression))) if compression is not None else ())
                 + (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
                 + (
                     tuple(shlex.quote(option) for option in database['options'].split(' '))
@@ -222,7 +225,7 @@ def dump_data_sources(
                 execute_command(
                     command,
                     shell=True,
-                    extra_environment=extra_environment,
+                    environment=environment,
                 )
             else:
                 dump.create_named_pipe_for_dump(dump_filename)
@@ -230,7 +233,7 @@ def dump_data_sources(
                     execute_command(
                         command,
                         shell=True,
-                        extra_environment=extra_environment,
+                        environment=environment,
                         run_to_completion=False,
                     )
                 )
@@ -238,7 +241,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'postgresql_databases')
+                os.path.join(borgmatic_runtime_directory, 'postgresql_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 
@@ -309,8 +313,11 @@ def restore_data_source_dump(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        connection_params['username']
-        or data_source.get('restore_username', data_source.get('username'))
+        (
+            connection_params['username']
+            or data_source.get('restore_username', data_source.get('username'))
+        ),
+        config,
     )
 
     all_databases = bool(data_source['name'] == 'all')
@@ -363,9 +370,7 @@ def restore_data_source_dump(
         )
     )
 
-    extra_environment = make_extra_environment(
-        data_source, restore_connection_params=connection_params
-    )
+    environment = make_environment(data_source, config, restore_connection_params=connection_params)
 
     logger.debug(f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}")
     if dry_run:
@@ -378,6 +383,6 @@ def restore_data_source_dump(
         [extract_process] if extract_process else [],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout if extract_process else None,
-        extra_environment=extra_environment,
+        environment=environment,
     )
-    execute_command(analyze_command, extra_environment=extra_environment)
+    execute_command(analyze_command, environment=environment)

borgmatic/hooks/data_source/snapshot.py

@@ -11,10 +11,10 @@ def get_contained_patterns(parent_directory, candidate_patterns):
     paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then
     /var is what we want to snapshot.
 
-    For this to work, a candidate pattern path can't have any globs or other non-literal characters
-    in the initial portion of the path that matches the parent directory. For instance, a parent
-    directory of /var would match a candidate pattern path of /var/log/*/data, but not a pattern
-    path like /v*/log/*/data.
+    For this function to work, a candidate pattern path can't have any globs or other non-literal
+    characters in the initial portion of the path that matches the parent directory. For instance, a
+    parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a
+    pattern path like /v*/log/*/data.
 
     The one exception is that if a regular expression pattern path starts with "^", that will get
     stripped off for purposes of matching against a parent directory.
@@ -31,8 +31,10 @@ def get_contained_patterns(parent_directory, candidate_patterns):
         candidate
         for candidate in candidate_patterns
         for candidate_path in (pathlib.PurePath(candidate.path.lstrip('^')),)
-        if pathlib.PurePath(parent_directory) == candidate_path
-        or pathlib.PurePath(parent_directory) in candidate_path.parents
+        if (
+            pathlib.PurePath(parent_directory) == candidate_path
+            or pathlib.PurePath(parent_directory) in candidate_path.parents
+        )
     )
     candidate_patterns -= set(contained_patterns)
 

borgmatic/hooks/data_source/sqlite.py

@@ -90,7 +90,8 @@ def dump_data_sources(
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(
-                os.path.join(borgmatic_runtime_directory, 'sqlite_databases')
+                os.path.join(borgmatic_runtime_directory, 'sqlite_databases'),
+                source=borgmatic.borg.pattern.Pattern_source.HOOK,
             )
         )
 

borgmatic/hooks/data_source/zfs.py

@@ -1,5 +1,6 @@
 import collections
 import glob
+import hashlib
 import logging
 import os
 import shutil
@@ -38,6 +39,9 @@ def get_datasets_to_backup(zfs_command, patterns):
     pattern paths represent the requested datasets to snapshot. But also include any datasets tagged
     with a borgmatic-specific user property, whether or not they appear in the patterns.
 
+    Only include datasets that contain at least one root pattern sourced from borgmatic
+    configuration (as opposed to generated elsewhere in borgmatic).
+
     Return the result as a sequence of Dataset instances, sorted by mount point.
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
@@ -48,7 +52,7 @@ def get_datasets_to_backup(zfs_command, patterns):
             '-t',
             'filesystem',
             '-o',
-            f'name,mountpoint,{BORGMATIC_USER_PROPERTY}',
+            f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}',
         )
     )
 
@@ -60,7 +64,12 @@ def get_datasets_to_backup(zfs_command, patterns):
             (
                 Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ())
                 for line in list_output.splitlines()
-                for (dataset_name, mount_point, user_property_value) in (line.rstrip().split('\t'),)
+                for (dataset_name, mount_point, can_mount, user_property_value) in (
+                    line.rstrip().split('\t'),
+                )
+                # Skip datasets that are marked "canmount=off", because mounting their snapshots will
+                # result in completely empty mount points—thereby preventing us from backing them up.
+                if can_mount == 'on'
             ),
             key=lambda dataset: dataset.mount_point,
             reverse=True,
@@ -83,7 +92,12 @@ def get_datasets_to_backup(zfs_command, patterns):
                 for contained_patterns in (
                     (
                         (
-                            (borgmatic.borg.pattern.Pattern(dataset.mount_point),)
+                            (
+                                borgmatic.borg.pattern.Pattern(
+                                    dataset.mount_point,
+                                    source=borgmatic.borg.pattern.Pattern_source.HOOK,
+                                ),
+                            )
                             if dataset.auto_backup
                             else ()
                         )
@@ -92,7 +106,12 @@ def get_datasets_to_backup(zfs_command, patterns):
                         )
                     ),
                 )
-                if contained_patterns
+                if dataset.auto_backup
+                or any(
+                    pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                    and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
+                    for pattern in contained_patterns
+                )
             ),
             key=lambda dataset: dataset.mount_point,
         )
@@ -115,7 +134,16 @@ def get_all_dataset_mount_points(zfs_command):
         )
     )
 
-    return tuple(sorted(line.rstrip() for line in list_output.splitlines()))
+    return tuple(
+        sorted(
+            {
+                mount_point
+                for line in list_output.splitlines()
+                for mount_point in (line.rstrip(),)
+                if mount_point != 'none'
+            }
+        )
+    )
 
 
 def snapshot_dataset(zfs_command, full_snapshot_name):  # pragma: no cover
@@ -155,10 +183,14 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
     )
 
 
-def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
+MOUNT_POINT_HASH_LENGTH = 10
+
+
+def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory):
     '''
-    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
-    path rewritten to be in a snapshot directory based on the given runtime directory.
+    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and the Dataset containing it,
+    return a new Pattern with its path rewritten to be in a snapshot directory based on both the
+    given runtime directory and the given Dataset's mount point.
 
     Move any initial caret in a regular expression pattern path to the beginning, so as not to break
     the regular expression.
@@ -173,6 +205,10 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
     rewritten_path = initial_caret + os.path.join(
         normalized_runtime_directory,
         'zfs_snapshots',
+        # Including this hash prevents conflicts between snapshot patterns for different datasets.
+        # For instance, without this, snapshotting a dataset at /var and another at /var/spool would
+        # result in overlapping snapshot patterns and therefore colliding mount attempts.
+        hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
         '.',  # Borg 1.4+ "slashdot" hack.
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
@@ -183,6 +219,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
         pattern.type,
         pattern.style,
         pattern.device,
+        source=borgmatic.borg.pattern.Pattern_source.HOOK,
     )
 
 
@@ -209,7 +246,8 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'Snapshotting ZFS datasets{dry_run_label}')
 
-    # List ZFS datasets to get their mount points.
+    # List ZFS datasets to get their mount points, but only consider those patterns that came from
+    # actual user configuration (as opposed to, say, other hooks).
     zfs_command = hook_config.get('zfs_command', 'zfs')
     requested_datasets = get_datasets_to_backup(zfs_command, patterns)
 
@@ -234,6 +272,9 @@ def dump_data_sources(
         snapshot_mount_path = os.path.join(
             normalized_runtime_directory,
             'zfs_snapshots',
+            hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(
+                MOUNT_POINT_HASH_LENGTH
+            ),
             dataset.mount_point.lstrip(os.path.sep),
         )
 
@@ -249,7 +290,9 @@ def dump_data_sources(
         )
 
         for pattern in dataset.contained_patterns:
-            snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
+            snapshot_pattern = make_borg_snapshot_pattern(
+                pattern, dataset, normalized_runtime_directory
+            )
 
             # Attempt to update the pattern in place, since pattern order matters to Borg.
             try:
@@ -334,6 +377,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
             os.path.normpath(borgmatic_runtime_directory),
         ),
         'zfs_snapshots',
+        '*',
     )
     logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
     umount_command = hook_config.get('umount_command', 'umount')
@@ -346,7 +390,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
         # child datasets before the shorter mount point paths of parent datasets.
         for mount_point in reversed(dataset_mount_points):
             snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep))
-            if not os.path.isdir(snapshot_mount_path):
+
+            # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
+            # dataset and therefore there's nothing to unmount.
+            if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
                 continue
 
             # This might fail if the path is already mounted, but we swallow errors here since we'll
@@ -370,10 +417,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                     return
                 except subprocess.CalledProcessError as error:
                     logger.debug(error)
-                    return
+                    continue
 
         if not dry_run:
-            shutil.rmtree(snapshots_directory)
+            shutil.rmtree(snapshot_mount_path, ignore_errors=True)
 
     # Destroy snapshots.
     full_snapshot_names = get_all_snapshots(zfs_command)

borgmatic/hooks/monitoring/ntfy.py

@@ -51,13 +51,13 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
 
         try:
             username = borgmatic.hooks.credential.parse.resolve_credential(
-                hook_config.get('username')
+                hook_config.get('username'), config
             )
             password = borgmatic.hooks.credential.parse.resolve_credential(
-                hook_config.get('password')
+                hook_config.get('password'), config
             )
             access_token = borgmatic.hooks.credential.parse.resolve_credential(
-                hook_config.get('access_token')
+                hook_config.get('access_token'), config
             )
         except ValueError as error:
             logger.warning(f'Ntfy credential error: {error}')

borgmatic/hooks/monitoring/pagerduty.py

@@ -42,7 +42,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
 
     try:
         integration_key = borgmatic.hooks.credential.parse.resolve_credential(
-            hook_config.get('integration_key')
+            hook_config.get('integration_key'), config
         )
     except ValueError as error:
         logger.warning(f'PagerDuty credential error: {error}')

borgmatic/hooks/monitoring/pushover.py

@@ -35,8 +35,10 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     state_config = hook_config.get(state.name.lower(), {})
 
     try:
-        token = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('token'))
-        user = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('user'))
+        token = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('token'), config
+        )
+        user = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('user'), config)
     except ValueError as error:
         logger.warning(f'Pushover credential error: {error}')
         return

borgmatic/hooks/monitoring/uptime_kuma.py

@@ -37,7 +37,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     logging.getLogger('urllib3').setLevel(logging.ERROR)
 
     try:
-        response = requests.get(f'{push_url}?{query}')
+        response = requests.get(f'{push_url}?{query}', verify=hook_config.get('verify_tls', True))
         if not response.ok:
             response.raise_for_status()
     except requests.exceptions.RequestException as error:

borgmatic/hooks/monitoring/zabbix.py

@@ -16,6 +16,42 @@ def initialize_monitor(
     pass
 
 
+def send_zabbix_request(server, headers, data):
+    '''
+    Given a Zabbix server URL, HTTP headers as a dict, and valid Zabbix JSON payload data as a dict,
+    send a request to the Zabbix server via API.
+
+    Return the response "result" value or None.
+    '''
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+
+    logger.debug(f'Sending a "{data["method"]}" request to the Zabbix server')
+
+    try:
+        response = requests.post(server, headers=headers, json=data)
+
+        if not response.ok:
+            response.raise_for_status()
+    except requests.exceptions.RequestException as error:
+        logger.warning(f'Zabbix error: {error}')
+
+        return None
+
+    try:
+        result = response.json().get('result')
+        error_message = result['data'][0]['error']
+    except requests.exceptions.JSONDecodeError:
+        logger.warning('Zabbix error: Cannot parse API response')
+
+        return None
+    except (TypeError, KeyError, IndexError):
+        return result
+    else:
+        logger.warning(f'Zabbix error: {error_message}')
+
+        return None
+
+
 def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
     '''
     Update the configured Zabbix item using either the itemid, or a host and key.
@@ -37,11 +73,18 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     )
 
     try:
-        username = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('username'))
-        password = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('password'))
-        api_key = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('api_key'))
+        username = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('username'), config
+        )
+        password = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('password'), config
+        )
+        api_key = borgmatic.hooks.credential.parse.resolve_credential(
+            hook_config.get('api_key'), config
+        )
     except ValueError as error:
         logger.warning(f'Zabbix credential error: {error}')
+
         return
 
     server = hook_config.get('server')
@@ -51,13 +94,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     value = state_config.get('value')
     headers = {'Content-Type': 'application/json-rpc'}
 
-    logger.info(f'Updating Zabbix{dry_run_label}')
+    logger.info(f'Pinging Zabbix{dry_run_label}')
     logger.debug(f'Using Zabbix URL: {server}')
 
-    if server is None:
-        logger.warning('Server missing for Zabbix')
-        return
-
     # Determine the Zabbix method used to store the value: itemid or host/key
     if itemid is not None:
         logger.info(f'Updating {itemid} on Zabbix')
@@ -68,8 +107,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
             'id': 1,
         }
 
-    elif (host and key) is not None:
-        logger.info(f'Updating Host:{host} and Key:{key} on Zabbix')
+    elif host is not None and key is not None:
+        logger.info(f'Updating Host: "{host}" and Key: "{key}" on Zabbix')
         data = {
             'jsonrpc': '2.0',
             'method': 'history.push',
@@ -79,58 +118,63 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
 
     elif host is not None:
         logger.warning('Key missing for Zabbix')
-        return
 
+        return
     elif key is not None:
         logger.warning('Host missing for Zabbix')
+
         return
     else:
         logger.warning('No Zabbix itemid or host/key provided')
+
         return
 
     # Determine the authentication method: API key or username/password
     if api_key is not None:
         logger.info('Using API key auth for Zabbix')
-        headers['Authorization'] = 'Bearer ' + api_key
-
-    elif (username and password) is not None:
-        logger.info('Using user/pass auth with user {username} for Zabbix')
-        auth_data = {
+        headers['Authorization'] = f'Bearer {api_key}'
+    elif username is not None and password is not None:
+        logger.info(f'Using user/pass auth with user {username} for Zabbix')
+        login_data = {
             'jsonrpc': '2.0',
             'method': 'user.login',
             'params': {'username': username, 'password': password},
             'id': 1,
         }
+
         if not dry_run:
-            logging.getLogger('urllib3').setLevel(logging.ERROR)
-            try:
-                response = requests.post(server, headers=headers, json=auth_data)
-                data['auth'] = response.json().get('result')
-                if not response.ok:
-                    response.raise_for_status()
-            except requests.exceptions.RequestException as error:
-                logger.warning(f'Zabbix error: {error}')
+            result = send_zabbix_request(server, headers, login_data)
+
+            if not result:
                 return
 
+            headers['Authorization'] = f'Bearer {result}'
     elif username is not None:
         logger.warning('Password missing for Zabbix authentication')
-        return
 
+        return
     elif password is not None:
         logger.warning('Username missing for Zabbix authentication')
+
         return
     else:
         logger.warning('Authentication data missing for Zabbix')
+
         return
 
     if not dry_run:
-        logging.getLogger('urllib3').setLevel(logging.ERROR)
-        try:
-            response = requests.post(server, headers=headers, json=data)
-            if not response.ok:
-                response.raise_for_status()
-        except requests.exceptions.RequestException as error:
-            logger.warning(f'Zabbix error: {error}')
+        send_zabbix_request(server, headers, data)
+
+    if username is not None and password is not None:
+        logout_data = {
+            'jsonrpc': '2.0',
+            'method': 'user.logout',
+            'params': [],
+            'id': 1,
+        }
+
+        if not dry_run:
+            send_zabbix_request(server, headers, logout_data)
 
 
 def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run):  # pragma: no cover

borgmatic/signals.py

@@ -24,6 +24,9 @@ def handle_signal(signal_number, frame):
         logger.critical('Exiting due to TERM signal')
         sys.exit(EXIT_CODE_FROM_SIGNAL + signal.SIGTERM)
     elif signal_number == signal.SIGINT:
+        # Borg doesn't always exit on a SIGINT, so give it a little encouragement.
+        os.killpg(os.getpgrp(), signal.SIGTERM)
+
         raise KeyboardInterrupt()
 
 

docs/fetch-contributors

@@ -26,8 +26,7 @@ def list_merged_pulls(url):
 
 
 def list_contributing_issues(url):
-    # labels = bug, design finalized, etc.
-    response = requests.get(f'{url}?labels=19,20,22,23,32,52,53,54', headers={'Accept': 'application/json', 'Content-Type': 'application/json'})
+    response = requests.get(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'})
 
     if not response.ok:
         response.raise_for_status()
@@ -39,7 +38,7 @@ PULLS_API_ENDPOINT_URLS = (
     'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/pulls',
     'https://api.github.com/repos/borgmatic-collective/borgmatic/pulls',
 )
-ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues'
+ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues?state=all'
 RECENT_CONTRIBUTORS_CUTOFF_DAYS = 365
 
 

docs/how-to/backup-your-databases.md

@@ -309,10 +309,8 @@ hooks:
 ### External passwords
 
 If you don't want to keep your database passwords in your borgmatic
-configuration file, you can instead pass them in via [environment
-variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/)
-or command-line [configuration
-overrides](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides).
+configuration file, you can instead pass them in [from external credential
+sources](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).
 
 
 ### Configuration backups

docs/how-to/monitor-your-backups.md

@@ -710,7 +710,10 @@ zabbix:
         - fail
 ```
 
-This hook requires the Zabbix server be running version 7.0+
+This hook requires the Zabbix server be running version 7.0.
+
+<span class="minilink minilink-addedin">New in version 1.9.3</span> Zabbix 7.2+
+is supported as well.
 
 
 ### Authentication methods

docs/how-to/provide-your-passwords.md

@@ -19,6 +19,7 @@ encryption_passphrase: yourpassphrase
 But if you'd rather store them outside of borgmatic, whether for convenience
 or security reasons, read on.
 
+
 ### Delegating to another application
 
 borgmatic supports calling another application such as a password manager to 
@@ -31,26 +32,17 @@ to provide the passphrase:
 encryption_passcommand: pass path/to/borg-passphrase
 ```
 
-Another example for [KeePassXC](https://keepassxc.org/):
-
-```yaml
-encryption_passcommand: keepassxc-cli show --show-protected --attributes Password credentials.kdbx borg_passphrase
-```
-
-... where `borg_passphrase` is the title of the KeePassXC entry containing your
-Borg encryption passphrase in its `Password` field.
-
 <span class="minilink minilink-addedin">New in version 1.9.9</span> Instead of
 letting Borg run the passcommand—potentially multiple times since borgmatic runs
 Borg multiple times—borgmatic now runs the passcommand itself and passes the
-resulting passprhase securely to Borg via an anonymous pipe. This means you
+resulting passphrase securely to Borg via an anonymous pipe. This means you
 should only ever get prompted for your password manager's passphrase at most
 once per borgmatic run.
 
 
-### Using systemd service credentials
+### systemd service credentials
 
-borgmatic supports using encrypted [systemd
+borgmatic supports reading encrypted [systemd
 credentials](https://systemd.io/CREDENTIALS/). To use this feature, start by
 saving your password as an encrypted credential to
 `/etc/credstore.encrypted/borgmatic.pw`, e.g.,
@@ -146,13 +138,172 @@ The one exception is `borgmatic config validate`, which doesn't actually load
 any credentials and should continue working anywhere.
 
 
+### Container secrets
+
+<span class="minilink minilink-addedin">New in version 1.9.11</span> When
+running inside a container, borgmatic can read [Docker
+secrets](https://docs.docker.com/compose/how-tos/use-secrets/) and [Podman
+secrets](https://www.redhat.com/en/blog/new-podman-secrets-command). Creating
+those secrets and passing them into your borgmatic container is outside the
+scope of this documentation, but here's a simple example of that with [Docker
+Compose](https://docs.docker.com/compose/):
+
+```yaml
+services:
+  borgmatic:
+    # Use the actual image name of your borgmatic container here.
+    image: borgmatic:latest
+    secrets:
+      - borgmatic_passphrase
+secrets:
+  borgmatic_passphrase:
+    file: /etc/borgmatic/passphrase.txt
+```
+
+This assumes there's a file on the host at `/etc/borgmatic/passphrase.txt`
+containing your passphrase. Docker or Podman mounts the contents of that file
+into a secret named `borgmatic_passphrase` in the borgmatic container at
+`/run/secrets/`.
+
+Once your container secret is in place, you can consume it within your borgmatic
+configuration file:
+
+```yaml
+encryption_passphrase: "{credential container borgmatic_passphrase}"
+```
+
+This reads the secret securely from a file mounted at
+`/run/secrets/borgmatic_passphrase` within the borgmatic container.
+
+The `{credential ...}` syntax works for several different options in a borgmatic
+configuration file besides just `encryption_passphrase`. For instance, the
+username, password, and API token options within database and monitoring hooks
+support `{credential ...}`:
+
+```yaml
+postgresql_databases:
+    - name: invoices
+      username: postgres
+      password: "{credential container borgmatic_db1}"
+```
+
+For specifics about which options are supported, see the
+[configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/).
+
+You can also optionally override the `/run/secrets` directory that borgmatic reads secrets from
+inside a container:
+
+```yaml
+container:
+    secrets_directory: /path/to/secrets
+```
+
+But you should only need to do this for development or testing purposes.
+
+
+### KeePassXC passwords
+
+<span class="minilink minilink-addedin">New in version 1.9.11</span> borgmatic
+supports reading passwords from the [KeePassXC](https://keepassxc.org/) password
+manager. To use this feature, start by creating an entry in your KeePassXC
+database, putting your password into the "Password" field of that entry and
+making sure it's saved.
+
+Then, you can consume that password in your borgmatic configuration file. For
+instance, if the entry's title is "borgmatic" and your KeePassXC database is
+located at `/etc/keys.kdbx`, do this:
+
+```yaml
+encryption_passphrase: "{credential keepassxc /etc/keys.kdbx borgmatic}"
+```
+
+But if the entry's title is multiple words like `borg pw`, you'll
+need to quote it:
+
+```yaml
+encryption_passphrase: "{credential keepassxc /etc/keys.kdbx 'borg pw'}"
+```
+
+With this in place, borgmatic runs the `keepassxc-cli` command to retrieve the
+passphrase on demand. But note that `keepassxc-cli` will prompt for its own
+passphrase in order to unlock its database, so be prepared to enter it when
+running borgmatic.
+
+The `{credential ...}` syntax works for several different options in a borgmatic
+configuration file besides just `encryption_passphrase`. For instance, the
+username, password, and API token options within database and monitoring hooks
+support `{credential ...}`:
+
+```yaml
+postgresql_databases:
+    - name: invoices
+      username: postgres
+      password: "{credential keepassxc /etc/keys.kdbx database}"
+```
+
+For specifics about which options are supported, see the
+[configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/).
+
+You can also optionally override the `keepassxc-cli` command that borgmatic calls to load
+passwords:
+
+```yaml
+keepassxc:
+    keepassxc_cli_command: /usr/local/bin/keepassxc-cli
+```
+
+
+### File-based credentials
+
+<span class="minilink minilink-addedin">New in version 1.9.11</span> borgmatic
+supports reading credentials from arbitrary file paths. To use this feature,
+start by writing your credential into a file that borgmatic has permission to
+read. Take care not to include anything in the file other than your credential.
+(borgmatic is smart enough to strip off a trailing newline though.)
+
+You can consume that credential file in your borgmatic configuration. For
+instance, if your credential file is at `/credentials/borgmatic.txt`, do this:
+
+```yaml
+encryption_passphrase: "{credential file /credentials/borgmatic.txt}"
+```
+
+With this in place, borgmatic reads the credential from the file path.
+
+The `{credential ...}` syntax works for several different options in a borgmatic
+configuration file besides just `encryption_passphrase`. For instance, the
+username, password, and API token options within database and monitoring hooks
+support `{credential ...}`:
+
+```yaml
+postgresql_databases:
+    - name: invoices
+      username: postgres
+      password: "{credential file /credentials/database.txt}"
+```
+
+For specifics about which options are supported, see the
+[configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/).
+
+
 ### Environment variable interpolation
 
 <span class="minilink minilink-addedin">New in version 1.6.4</span> borgmatic
 supports interpolating arbitrary environment variables directly into option
 values in your configuration file. That means you can instruct borgmatic to
 pull your repository passphrase, your database passwords, or any other option
-values from environment variables. For instance:
+values from environment variables.
+
+Be aware though that environment variables may be less secure than some of the
+other approaches above for getting credentials into borgmatic. That's because
+environment variables may be visible from within child processes and/or OS-level
+process metadata.
+
+Here's an example of using an environment variable from borgmatic's
+configuration file:
 
 ```yaml
 encryption_passphrase: ${YOUR_PASSPHRASE}
@@ -214,6 +365,7 @@ can escape it with a backslash. For instance, if your password is literally
 encryption_passphrase: \${A}@!
 ```
 
+
 ## Related features
 
 Another way to override particular options within a borgmatic configuration
@@ -226,9 +378,3 @@ Additionally, borgmatic action hooks support their own [variable
 interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation),
 although in that case it's for particular borgmatic runtime values rather than
 (only) environment variables.
-
-Lastly, if you do want to specify your passhprase directly within borgmatic
-configuration, but you'd like to keep it in a separate file from your main
-configuration, you can [use a configuration include or a merge
-include](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-includes)
-to pull in an external password.

docs/how-to/snapshot-your-filesystems.md

@@ -54,8 +54,8 @@ You have a couple of options for borgmatic to find and backup your ZFS datasets:
  * For any dataset you'd like backed up, add its mount point to borgmatic's
    `source_directories` option.
  * <span class="minilink minilink-addedin">New in version 1.9.6</span> Or
-   include the mount point with borgmatic's `patterns` or `patterns_from`
-   options.
+   include the mount point as a root pattern with borgmatic's `patterns` or
+   `patterns_from` options.
  * Or set the borgmatic-specific user property
    `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs
    set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find
@@ -65,6 +65,11 @@ If you have multiple borgmatic configuration files with ZFS enabled, and you'd
 like particular datasets to be backed up only for particular configuration
 files, use the `source_directories` option instead of the user property.
 
+<span class="minilink minilink-addedin">New in version 1.9.11</span> borgmatic
+won't snapshot datasets with the `canmount=off` property, which is often set on
+datasets that only serve as a container for other datasets. Use `zfs get
+canmount datasetname` to see the `canmount` value for a dataset.
+
 During a backup, borgmatic automatically snapshots these discovered datasets
 (non-recursively), temporarily mounts the snapshots within its [runtime
 directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory),
@@ -147,7 +152,8 @@ For any subvolume you'd like backed up, add its path to borgmatic's
 `source_directories` option.
 
 <span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
-the mount point with borgmatic's `patterns` or `patterns_from` options.
+the mount point as a root pattern with borgmatic's `patterns` or `patterns_from`
+options.
 
 During a backup, borgmatic snapshots these subvolumes (non-recursively) and
 includes the snapshotted files in the paths sent to Borg. borgmatic is also
@@ -199,6 +205,14 @@ Volume Manager) and sending those snapshots to Borg for backup. LVM isn't
 itself a filesystem, but it can take snapshots at the layer right below your
 filesystem.
 
+Note that, due to Borg being a file-level backup, this feature is really only
+suitable for filesystems, not whole disk or raw images containing multiple
+filesystems (for example, if you're using a LVM volume to run a Windows
+KVM that contains an MBR, partitions, etc.).
+
+In those cases, you can omit the `lvm:` option and use Borg's own support for
+[image backup](https://borgbackup.readthedocs.io/en/stable/deployment/image-backup.html).
+
 To use this feature, first you need one or more mounted LVM logical volumes.
 Then, enable LVM within borgmatic by adding the following line to your
 configuration file:
@@ -252,7 +266,8 @@ For any logical volume you'd like backed up, add its mount point to
 borgmatic's `source_directories` option.
 
 <span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
-the mount point with borgmatic's `patterns` or `patterns_from` options.
+the mount point as a root pattern with borgmatic's `patterns` or `patterns_from`
+options.
 
 During a backup, borgmatic automatically snapshots these discovered logical volumes
 (non-recursively), temporarily mounts the snapshots within its [runtime

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "1.9.10"
+version = "1.9.13"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]

tests/end-to-end/commands/fake_keepassxc_cli.py

@@ -0,0 +1,29 @@
+import argparse
+import sys
+
+
+def parse_arguments(*unparsed_arguments):
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument('command')
+    parser.add_argument('--show-protected', action='store_true')
+    parser.add_argument('--attributes')
+    parser.add_argument('database_path')
+    parser.add_argument('attribute_name')
+
+    return parser.parse_args(unparsed_arguments)
+
+
+def main():
+    arguments = parse_arguments(*sys.argv[1:])
+
+    assert arguments.command == 'show'
+    assert arguments.show_protected
+    assert arguments.attributes == 'Password'
+    assert arguments.database_path.endswith('.kdbx')
+    assert arguments.attribute_name
+
+    print('test')
+
+
+if __name__ == '__main__':
+    main()

tests/end-to-end/commands/fake_zfs.py

@@ -27,6 +27,7 @@ BUILTIN_DATASETS = (
         'used': '256K',
         'avail': '23.7M',
         'refer': '25K',
+        'canmount': 'on',
         'mountpoint': '/pool',
     },
     {
@@ -34,6 +35,7 @@ BUILTIN_DATASETS = (
         'used': '256K',
         'avail': '23.7M',
         'refer': '25K',
+        'canmount': 'on',
         'mountpoint': '/pool/dataset',
     },
 )

tests/end-to-end/hooks/credential/test_container.py

@@ -0,0 +1,68 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path, secrets_directory):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase loaded from container secrets in the given secrets
+    directory.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "{credential container mysecret}"'
+        + f'\ncontainer:\n    secrets_directory: {secrets_directory}'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_container_secret():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    os.chdir(temporary_directory)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path, secrets_directory=temporary_directory)
+
+        secret_path = os.path.join(temporary_directory, 'mysecret')
+        with open(secret_path, 'w') as secret_file:
+            secret_file.write('test')
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/end-to-end/hooks/credential/test_file.py

@@ -0,0 +1,68 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path, credential_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase loaded from file at the given credential path.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "{credential file '
+        + credential_path
+        + '}"'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_file_credential():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    os.chdir(temporary_directory)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        credential_path = os.path.join(temporary_directory, 'mycredential')
+        generate_configuration(config_path, repository_path, credential_path)
+
+        with open(credential_path, 'w') as credential_file:
+            credential_file.write('test')
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/end-to-end/hooks/credential/test_keepassxc.py

@@ -0,0 +1,67 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase loaded from keepassxc-cli.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "{credential keepassxc keys.kdbx mypassword}"'
+        + '\nkeepassxc:\n    keepassxc_cli_command: python3 /app/tests/end-to-end/commands/fake_keepassxc_cli.py'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_keepassxc_password():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    os.chdir(temporary_directory)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path)
+
+        database_path = os.path.join(temporary_directory, 'keys.kdbx')
+        with open(database_path, 'w') as database_file:
+            database_file.write('fake KeePassXC database to pacify file existence check')
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/end-to-end/hooks/credential/test_systemd.py

@@ -30,15 +30,13 @@ def generate_configuration(config_path, repository_path):
     config_file.close()
 
 
-def test_borgmatic_command():
+def test_systemd_credential():
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()
     repository_path = os.path.join(temporary_directory, 'test.borg')
-    extract_path = os.path.join(temporary_directory, 'extract')
 
     original_working_directory = os.getcwd()
-    os.mkdir(extract_path)
-    os.chdir(extract_path)
+    os.chdir(temporary_directory)
 
     try:
         config_path = os.path.join(temporary_directory, 'test.yaml')

tests/unit/actions/test_create.py

@@ -5,16 +5,21 @@ import pytest
 from flexmock import flexmock
 
 from borgmatic.actions import create as module
-from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
+from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
 
 
 @pytest.mark.parametrize(
     'pattern_line,expected_pattern',
     (
-        ('R /foo', Pattern('/foo')),
-        ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)),
-        ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)),
-        ('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)),
+        ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)),
+        ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)),
+        ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)),
+        (
+            '+ sh:/foo*',
+            Pattern(
+                '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG
+            ),
+        ),
     ),
 )
 def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern):
@@ -28,8 +33,8 @@ def test_parse_pattern_with_invalid_pattern_line_errors():
 
 def test_collect_patterns_converts_source_directories():
     assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == (
-        Pattern('/foo'),
-        Pattern('/bar'),
+        Pattern('/foo', source=Pattern_source.CONFIG),
+        Pattern('/bar', source=Pattern_source.CONFIG),
     )
 
 
@@ -48,9 +53,15 @@ def test_collect_patterns_parses_config_patterns():
 
 def test_collect_patterns_converts_exclude_patterns():
     assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
-        Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
-        Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
-        Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL),
+        Pattern(
+            '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
+        ),
+        Pattern(
+            '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
+        ),
+        Pattern(
+            '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG
+        ),
     )
 
 

tests/unit/borg/test_borg.py

@@ -17,7 +17,7 @@ def test_run_arbitrary_borg_calls_borg_with_flags():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -41,7 +41,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag():
         ('borg', 'break-lock', '--info', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -66,7 +66,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag():
         ('borg', 'break-lock', '--debug', '--show-rc', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -94,7 +94,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags():
         ('borg', 'break-lock', '--lock-wait', '5', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -118,7 +118,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag():
         ('borg', 'break-lock', "'::$ARCHIVE'"),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -143,7 +143,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
         ('borg1', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg1',
         borg_exit_codes=None,
@@ -169,7 +169,7 @@ def test_run_arbitrary_borg_with_exit_codes_calls_borg_using_them():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -195,7 +195,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags()
         ('borg', 'break-lock', '--remote-path', 'borg1', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -222,7 +222,7 @@ def test_run_arbitrary_borg_with_remote_path_injection_attack_gets_escaped():
         ('borg', 'break-lock', '--remote-path', "'borg1; naughty-command'", '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -247,7 +247,7 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg():
         ('borg', 'list', '--progress', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -271,7 +271,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -295,7 +295,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise():
         ('borg',),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -319,7 +319,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags
         ('borg', 'key', 'export', '--info', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -344,7 +344,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla
         ('borg', 'debug', 'dump-manifest', '--info', '::', 'path'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -371,7 +371,7 @@ def test_run_arbitrary_borg_calls_borg_with_working_directory():
         ('borg', 'break-lock', '::'),
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
         shell=True,
-        extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
+        environment={'BORG_REPO': 'repo', 'ARCHIVE': ''},
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_break_lock.py

@@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes
     )
     flexmock(module).should_receive('execute_command').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_change_passphrase.py

@@ -25,7 +25,7 @@ def insert_execute_command_mock(
         command,
         output_file=output_file,
         output_log_level=module.logging.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_check.py

@@ -18,7 +18,7 @@ def insert_execute_command_mock(
     flexmock(module).should_receive('execute_command').with_args(
         command,
         output_file=output_file,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,
@@ -342,7 +342,7 @@ def test_check_archives_with_progress_passes_through_to_borg():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--progress', 'repo'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -377,7 +377,7 @@ def test_check_archives_with_repair_passes_through_to_borg():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--repair', 'repo'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -412,7 +412,7 @@ def test_check_archives_with_max_duration_flag_passes_through_to_borg():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--max-duration', '33', 'repo'),
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -447,7 +447,7 @@ def test_check_archives_with_max_duration_option_passes_through_to_borg():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--max-duration', '33', 'repo'),
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -610,7 +610,7 @@ def test_check_archives_with_max_duration_flag_overrides_max_duration_option():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--max-duration', '44', 'repo'),
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -963,7 +963,7 @@ def test_check_archives_with_match_archives_passes_through_to_borg():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'check', '--match-archives', 'foo-*', 'repo'),
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_compact.py

@@ -17,7 +17,7 @@ def insert_execute_command_mock(
     flexmock(module).should_receive('execute_command').with_args(
         compact_command,
         output_log_level=output_log_level,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=compact_command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_create.py

@@ -4,7 +4,7 @@ import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import create as module
-from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
+from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
 
 from ..test_verbosity import insert_logging_mock
 
@@ -185,6 +185,12 @@ def test_any_parent_directories_treats_unrelated_paths_as_non_match():
 
 
 def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_list():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
     flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         'Processing files ...\n- /foo\n+ /bar\n- /baz'
@@ -204,6 +210,12 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_
 
 
 def test_collect_special_file_paths_skips_borgmatic_runtime_directory():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
     flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /run/borgmatic/bar\n- /baz'
@@ -231,6 +243,12 @@ def test_collect_special_file_paths_skips_borgmatic_runtime_directory():
 
 
 def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_from_paths_output_errors():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
     flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /bar\n- /baz'
@@ -251,6 +269,12 @@ def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_fro
 
 
 def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory_missing_from_paths_output_does_not_raise():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
     flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n- /bar\n- /baz'
@@ -270,6 +294,12 @@ def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory
 
 
 def test_collect_special_file_paths_excludes_non_special_files():
+    flexmock(module.flags).should_receive('omit_flag').replace_with(
+        lambda arguments, flag: arguments
+    )
+    flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
+        lambda arguments, flag: arguments
+    )
     flexmock(module.environment).should_receive('make_environment').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n+ /bar\n+ /baz'
@@ -290,30 +320,6 @@ def test_collect_special_file_paths_excludes_non_special_files():
     ) == ('/foo', '/baz')
 
 
-def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command():
-    flexmock(module.environment).should_receive('make_environment').and_return(None)
-    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
-        ('borg', 'create', '--dry-run', '--list'),
-        capture_stderr=True,
-        working_directory=None,
-        extra_environment=None,
-        borg_local_path='borg',
-        borg_exit_codes=None,
-    ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once()
-    flexmock(module).should_receive('special_file').and_return(True)
-    flexmock(module.os.path).should_receive('exists').and_return(False)
-    flexmock(module).should_receive('any_parent_directories').never()
-
-    module.collect_special_file_paths(
-        dry_run=False,
-        create_command=('borg', 'create', '--exclude-nodump'),
-        config={},
-        local_path='borg',
-        working_directory=None,
-        borgmatic_runtime_directory='/run/borgmatic',
-    )
-
-
 DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
 REPO_ARCHIVE = (f'repo::{DEFAULT_ARCHIVE_NAME}',)
 
@@ -663,6 +669,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
                 '/dev/null',
                 Pattern_type.NO_RECURSE,
                 Pattern_style.FNMATCH,
+                source=Pattern_source.INTERNAL,
             ),
         ),
         '/run/borgmatic',
@@ -713,6 +720,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
                 '/dev/null',
                 Pattern_type.NO_RECURSE,
                 Pattern_style.FNMATCH,
+                source=Pattern_source.INTERNAL,
             ),
         ),
         '/run/borgmatic',
@@ -969,7 +977,7 @@ def test_create_archive_calls_borg_with_parameters():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1003,7 +1011,7 @@ def test_create_archive_calls_borg_with_environment():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=environment,
+        environment=environment,
     )
 
     module.create_archive(
@@ -1036,7 +1044,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1066,7 +1074,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     )
@@ -1103,7 +1111,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -1133,7 +1141,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     )
@@ -1172,7 +1180,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1209,7 +1217,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory='/working/dir',
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1244,7 +1252,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them():
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1278,7 +1286,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1316,7 +1324,7 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1350,7 +1358,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     insert_logging_mock(logging.INFO)
 
@@ -1385,7 +1393,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1431,7 +1439,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         create_command,
@@ -1441,7 +1449,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(
@@ -1472,7 +1480,7 @@ def test_create_archive_with_json_calls_borg_with_json_flag():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     ).and_return('[]')
@@ -1506,7 +1514,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag():
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create', '--json') + REPO_ARCHIVE,
         working_directory=None,
-        extra_environment=None,
+        environment=None,
         borg_local_path='borg',
         borg_exit_codes=None,
     ).and_return('[]')
@@ -1547,7 +1555,7 @@ def test_create_archive_calls_borg_with_working_directory():
         borg_local_path='borg',
         borg_exit_codes=None,
         working_directory='/working/dir',
-        extra_environment=None,
+        environment=None,
     )
 
     module.create_archive(

tests/unit/borg/test_delete.py

@@ -370,9 +370,9 @@ def test_delete_archives_calls_borg_delete_with_working_directory():
     flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never()
     command = flexmock()
     flexmock(module).should_receive('make_delete_command').and_return(command)
-    extra_environment = flexmock()
+    environment = flexmock()
     flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
-        extra_environment
+        environment
     )
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         '/working/dir'
@@ -380,7 +380,7 @@ def test_delete_archives_calls_borg_delete_with_working_directory():
     flexmock(module.borgmatic.execute).should_receive('execute_command').with_args(
         command,
         output_log_level=logging.ANSWER,
-        extra_environment=extra_environment,
+        environment=environment,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_environment.py

@@ -4,6 +4,12 @@ from borgmatic.borg import environment as module
 
 
 def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_file_descriptor_in_environment():
+    flexmock(module.os).should_receive('environ').and_return(
+        {'USER': 'root', 'BORG_PASSCOMMAND': 'nope'}
+    )
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).and_return(None)
     flexmock(module.borgmatic.borg.passcommand).should_receive(
         'get_passphrase_from_passcommand'
     ).and_return('passphrase')
@@ -14,132 +20,153 @@ def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_fil
 
     environment = module.make_environment({'encryption_passcommand': 'command'})
 
-    assert not environment.get('BORG_PASSCOMMAND')
+    assert environment.get('BORG_PASSPHRASE') is None
+    assert environment.get('BORG_PASSCOMMAND') is None
     assert environment.get('BORG_PASSPHRASE_FD') == '3'
 
 
-def test_make_environment_with_passphrase_should_set_environment():
+def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_in_environment():
+    flexmock(module.os).should_receive('environ').and_return(
+        {'USER': 'root', 'BORG_PASSPHRASE': 'nope', 'BORG_PASSCOMMAND': 'nope'}
+    )
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
     flexmock(module.borgmatic.borg.passcommand).should_receive(
         'get_passphrase_from_passcommand'
     ).and_return(None)
-    flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
-    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
-        'resolve_credential'
-    ).replace_with(lambda value: value)
+    flexmock(module.os).should_receive('pipe').and_return((3, 4))
+    flexmock(module.os).should_receive('write')
+    flexmock(module.os).should_receive('close')
+    flexmock(module.os).should_receive('set_inheritable')
 
     environment = module.make_environment({'encryption_passphrase': 'pass'})
 
-    assert environment.get('BORG_PASSPHRASE') == 'pass'
+    assert environment.get('BORG_PASSPHRASE') is None
+    assert environment.get('BORG_PASSCOMMAND') is None
+    assert environment.get('BORG_PASSPHRASE_FD') == '3'
 
 
-def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_environment():
+def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_passphrase_file_descriptor_in_environment():
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    config = {'encryption_passphrase': '{credential systemd pass}'}
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential',
+    ).with_args('{credential systemd pass}', config).and_return('pass')
     flexmock(module.borgmatic.borg.passcommand).should_receive(
         'get_passphrase_from_passcommand'
-    ).and_return(None)
-    flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
-    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
-        'resolve_credential'
-    ).with_args('{credential systemd pass}').and_return('pass')
+    ).never()
+    flexmock(module.os).should_receive('pipe').and_return((3, 4))
+    flexmock(module.os).should_receive('write')
+    flexmock(module.os).should_receive('close')
+    flexmock(module.os).should_receive('set_inheritable')
 
-    environment = module.make_environment({'encryption_passphrase': '{credential systemd pass}'})
+    environment = module.make_environment(config)
 
-    assert environment.get('BORG_PASSPHRASE') == 'pass'
+    assert environment.get('BORG_PASSPHRASE') is None
+    assert environment.get('BORG_PASSPHRASE_FD') == '3'
 
 
 def test_make_environment_with_ssh_command_should_set_environment():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'ssh_command': 'ssh -C'})
 
     assert environment.get('BORG_RSH') == 'ssh -C'
 
 
 def test_make_environment_without_configuration_sets_certain_environment_variables():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({})
 
     # Default environment variables.
     assert environment == {
+        'USER': 'root',
         'BORG_EXIT_CODES': 'modern',
         'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
         'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
     }
 
 
-def test_make_environment_without_configuration_does_not_set_certain_environment_variables_if_already_set():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+def test_make_environment_without_configuration_passes_through_default_environment_variables_untouched():
+    flexmock(module.os).should_receive('environ').and_return(
+        {
+            'USER': 'root',
+            'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup',
+            'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah',
+        }
+    )
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').with_args(
-        'BORG_RELOCATED_REPO_ACCESS_IS_OK'
-    ).and_return('yup')
-    flexmock(module.os.environ).should_receive('get').with_args(
-        'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'
-    ).and_return('nah')
     environment = module.make_environment({})
 
-    assert environment == {'BORG_EXIT_CODES': 'modern'}
+    assert environment == {
+        'USER': 'root',
+        'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup',
+        'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah',
+        'BORG_EXIT_CODES': 'modern',
+    }
 
 
 def test_make_environment_with_relocated_repo_access_true_should_set_environment_yes():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'relocated_repo_access_is_ok': True})
 
     assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
 
 
 def test_make_environment_with_relocated_repo_access_false_should_set_environment_no():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'relocated_repo_access_is_ok': False})
 
     assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'no'
 
 
 def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environment_YES():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'check_i_know_what_i_am_doing': True})
 
     assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'YES'
 
 
 def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environment_NO():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'check_i_know_what_i_am_doing': False})
 
     assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'NO'
 
 
 def test_make_environment_with_integer_variable_value():
-    flexmock(module.borgmatic.borg.passcommand).should_receive(
-        'get_passphrase_from_passcommand'
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
     ).and_return(None)
     flexmock(module.os).should_receive('pipe').never()
-    flexmock(module.os.environ).should_receive('get').and_return(None)
     environment = module.make_environment({'borg_files_cache_ttl': 40})
+
     assert environment.get('BORG_FILES_CACHE_TTL') == '40'

tests/unit/borg/test_export_key.py

@@ -22,7 +22,7 @@ def insert_execute_command_mock(
         command,
         output_file=output_file,
         output_log_level=module.logging.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_export_tar.py

@@ -23,7 +23,7 @@ def insert_execute_command_mock(
         command,
         output_file=None if capture else module.DO_NOT_CAPTURE,
         output_log_level=output_log_level,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=borg_local_path,
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_extract.py

@@ -12,7 +12,7 @@ def insert_execute_command_mock(command, destination_path=None, borg_exit_codes=
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=destination_path,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,
@@ -587,7 +587,7 @@ def test_extract_archive_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', '--progress', 'repo::archive'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -639,7 +639,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
         ('borg', 'extract', '--stdout', 'repo::archive'),
         output_file=module.subprocess.PIPE,
         run_to_completion=False,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -674,7 +674,7 @@ def test_extract_archive_skips_abspath_for_remote_repository():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', 'server:repo::archive'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_flags.py

@@ -285,3 +285,45 @@ def test_warn_for_aggressive_archive_flags_with_glob_archives_and_json_missing_a
     flexmock(module.logger).should_receive('warning').never()
 
     module.warn_for_aggressive_archive_flags(('borg', '--glob-archives', 'foo*'), '{}')
+
+
+def test_omit_flag_removes_flag_from_arguments():
+    assert module.omit_flag(('borg', 'create', '--flag', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_without_flag_present_passes_through_arguments():
+    assert module.omit_flag(('borg', 'create', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_and_value_removes_flag_and_value_from_arguments():
+    assert module.omit_flag_and_value(
+        ('borg', 'create', '--flag', 'value', '--other'), '--flag'
+    ) == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_and_value_with_equals_sign_removes_flag_and_value_from_arguments():
+    assert module.omit_flag_and_value(('borg', 'create', '--flag=value', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )
+
+
+def test_omit_flag_and_value_without_flag_present_passes_through_arguments():
+    assert module.omit_flag_and_value(('borg', 'create', '--other'), '--flag') == (
+        'borg',
+        'create',
+        '--other',
+    )

tests/unit/borg/test_info.py

@@ -514,7 +514,7 @@ def test_display_archives_info_calls_borg_with_working_directory():
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         full_command=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,
@@ -523,7 +523,7 @@ def test_display_archives_info_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         full_command=object,
         output_log_level=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,

tests/unit/borg/test_list.py

@@ -353,7 +353,7 @@ def test_list_archive_calls_borg_with_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -419,7 +419,7 @@ def test_list_archive_calls_borg_with_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg2', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg2',
         borg_exit_codes=None,
@@ -469,7 +469,7 @@ def test_list_archive_calls_borg_using_exit_codes():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -507,7 +507,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -520,7 +520,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive1') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -528,7 +528,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive2') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -576,7 +576,7 @@ def test_list_archive_calls_borg_with_archive():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -699,7 +699,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -759,7 +759,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-list', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -808,7 +808,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--repo', 'repo', 'archive1') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -816,7 +816,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--repo', 'repo', 'archive2') + glob_paths,
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -875,7 +875,7 @@ def test_list_archive_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_mount.py

@@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes
     )
     flexmock(module).should_receive('execute_command').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=command[0],
         borg_exit_codes=borg_exit_codes,
@@ -262,7 +262,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'),
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -337,7 +337,7 @@ def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags
             'repo',
             '/mnt',
         ),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_passcommand.py

@@ -3,26 +3,13 @@ from flexmock import flexmock
 from borgmatic.borg import passcommand as module
 
 
-def test_run_passcommand_with_passphrase_configured_bails():
-    module.run_passcommand.cache_clear()
-    flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never()
-
-    assert (
-        module.run_passcommand('passcommand', passphrase_configured=True, working_directory=None)
-        is None
-    )
-
-
-def test_run_passcommand_without_passphrase_configured_executes_passcommand():
+def test_run_passcommand_does_not_raise():
     module.run_passcommand.cache_clear()
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
-    ).and_return('passphrase').once()
+    ).and_return('passphrase')
 
-    assert (
-        module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None)
-        == 'passphrase'
-    )
+    assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'
 
 
 def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it():
@@ -30,9 +17,9 @@ def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         '/working'
     )
-    flexmock(module).should_receive('run_passcommand').with_args(
-        'command', False, '/working'
-    ).and_return('passphrase').once()
+    flexmock(module).should_receive('run_passcommand').with_args('command', '/working').and_return(
+        'passphrase'
+    ).once()
 
     assert (
         module.get_passphrase_from_passcommand(
@@ -42,38 +29,10 @@ def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it():
     )
 
 
-def test_get_passphrase_from_passcommand_with_configured_passphrase_and_passcommand_detects_passphrase():
-    module.run_passcommand.cache_clear()
-    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
-        '/working'
-    )
-    flexmock(module).should_receive('run_passcommand').with_args(
-        'command', True, '/working'
-    ).and_return(None).once()
+def test_get_passphrase_from_passcommand_without_configured_passcommand_bails():
+    flexmock(module).should_receive('run_passcommand').never()
 
-    assert (
-        module.get_passphrase_from_passcommand(
-            {'encryption_passphrase': 'passphrase', 'encryption_passcommand': 'command'},
-        )
-        is None
-    )
-
-
-def test_get_passphrase_from_passcommand_with_configured_blank_passphrase_and_passcommand_detects_passphrase():
-    module.run_passcommand.cache_clear()
-    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
-        '/working'
-    )
-    flexmock(module).should_receive('run_passcommand').with_args(
-        'command', True, '/working'
-    ).and_return(None).once()
-
-    assert (
-        module.get_passphrase_from_passcommand(
-            {'encryption_passphrase': '', 'encryption_passcommand': 'command'},
-        )
-        is None
-    )
+    assert module.get_passphrase_from_passcommand({}) is None
 
 
 def test_run_passcommand_caches_passcommand_after_first_call():
@@ -82,11 +41,5 @@ def test_run_passcommand_caches_passcommand_after_first_call():
         'execute_command_and_capture_output'
     ).and_return('passphrase').once()
 
-    assert (
-        module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None)
-        == 'passphrase'
-    )
-    assert (
-        module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None)
-        == 'passphrase'
-    )
+    assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'
+    assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'

tests/unit/borg/test_prune.py

@@ -17,7 +17,7 @@ def insert_execute_command_mock(
     flexmock(module).should_receive('execute_command').with_args(
         prune_command,
         output_log_level=output_log_level,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=prune_command[0],
         borg_exit_codes=borg_exit_codes,
@@ -497,7 +497,7 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag
             'repo',
         ),
         output_log_level=logging.INFO,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_repo_create.py

@@ -36,7 +36,7 @@ def insert_repo_create_command_mock(
     flexmock(module).should_receive('execute_command').with_args(
         repo_create_command,
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=repo_create_command[0],
         borg_exit_codes=borg_exit_codes,

tests/unit/borg/test_repo_delete.py

@@ -290,7 +290,7 @@ def test_delete_repository_with_defaults_does_not_capture_output():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
-        extra_environment=object,
+        environment=object,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -319,7 +319,7 @@ def test_delete_repository_with_force_captures_output():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=None,
-        extra_environment=object,
+        environment=object,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -348,7 +348,7 @@ def test_delete_repository_with_cache_only_captures_output():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=None,
-        extra_environment=object,
+        environment=object,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -379,7 +379,7 @@ def test_delete_repository_calls_borg_with_working_directory():
         command,
         output_log_level=module.logging.ANSWER,
         output_file=module.borgmatic.execute.DO_NOT_CAPTURE,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_repo_info.py

@@ -24,7 +24,7 @@ def test_display_repository_info_calls_borg_with_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -33,7 +33,7 @@ def test_display_repository_info_calls_borg_with_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -60,7 +60,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -69,7 +69,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -101,7 +101,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -110,7 +110,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -142,7 +142,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -178,7 +178,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--debug', '--show-rc', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -187,7 +187,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--debug', '--show-rc', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -220,7 +220,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -256,7 +256,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -291,7 +291,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg1', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -300,7 +300,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg1',
         borg_exit_codes=None,
@@ -334,7 +334,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them():
     borg_exit_codes = flexmock()
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -343,7 +343,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -375,7 +375,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--remote-path', 'borg1', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -384,7 +384,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--remote-path', 'borg1', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -417,7 +417,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--umask', '077', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -426,7 +426,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--umask', '077', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -462,7 +462,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--log-json', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -471,7 +471,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--log-json', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -504,7 +504,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--lock-wait', '5', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -513,7 +513,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--lock-wait', '5', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -547,7 +547,7 @@ def test_display_repository_info_calls_borg_with_working_directory():
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'repo-info', '--json', '--repo', 'repo'),
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -556,7 +556,7 @@ def test_display_repository_info_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'repo-info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_repo_list.py

@@ -39,7 +39,7 @@ def test_resolve_archive_name_calls_borg_with_flags():
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
         borg_local_path='borg',
         borg_exit_codes=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
     ).and_return(expected_archive + '\n')
 
@@ -61,7 +61,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -86,7 +86,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -111,7 +111,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg1',
         borg_exit_codes=None,
@@ -137,7 +137,7 @@ def test_resolve_archive_name_with_exit_codes_calls_borg_using_them():
     borg_exit_codes = flexmock()
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -161,7 +161,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -186,7 +186,7 @@ def test_resolve_archive_name_with_umask_calls_borg_with_umask_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--umask', '077') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -209,7 +209,7 @@ def test_resolve_archive_name_without_archives_raises():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -232,7 +232,7 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -257,7 +257,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -285,7 +285,7 @@ def test_resolve_archive_name_calls_borg_with_working_directory():
         ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
         borg_local_path='borg',
         borg_exit_codes=None,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
     ).and_return(expected_archive + '\n')
 
@@ -773,7 +773,7 @@ def test_list_repository_calls_borg_with_working_directory():
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         full_command=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,
@@ -782,7 +782,7 @@ def test_list_repository_calls_borg_with_working_directory():
     flexmock(module).should_receive('execute_command').with_args(
         full_command=object,
         output_log_level=object,
-        extra_environment=object,
+        environment=object,
         working_directory='/working/dir',
         borg_local_path=object,
         borg_exit_codes=object,

tests/unit/borg/test_transfer.py

@@ -21,7 +21,7 @@ def test_transfer_archives_calls_borg_with_flags():
         ('borg', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -55,7 +55,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag():
         ('borg', 'transfer', '--repo', 'repo', '--dry-run'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -86,7 +86,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag():
         ('borg', 'transfer', '--info', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -117,7 +117,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag():
         ('borg', 'transfer', '--debug', '--show-rc', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -151,7 +151,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag():
         ('borg', 'transfer', '--match-archives', 'archive', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -184,7 +184,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl
         ('borg', 'transfer', '--match-archives', 'sh:foo*', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -217,7 +217,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv
         ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -248,7 +248,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path():
         ('borg2', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg2',
         borg_exit_codes=None,
@@ -281,7 +281,7 @@ def test_transfer_archives_with_exit_codes_calls_borg_using_them():
         ('borg', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=borg_exit_codes,
@@ -315,7 +315,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags():
         ('borg', 'transfer', '--remote-path', 'borg2', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -349,7 +349,7 @@ def test_transfer_archives_with_umask_calls_borg_with_umask_flags():
         ('borg', 'transfer', '--umask', '077', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -383,7 +383,7 @@ def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags():
         ('borg', 'transfer', '--log-json', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -418,7 +418,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags():
         ('borg', 'transfer', '--lock-wait', '5', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -449,7 +449,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag():
         ('borg', 'transfer', '--progress', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=module.DO_NOT_CAPTURE,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -484,7 +484,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name):
         ('borg', 'transfer', flag_name, 'value', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -521,7 +521,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla
         ('borg', 'transfer', '--repo', 'repo', '--other-repo', 'other'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -566,7 +566,7 @@ def test_transfer_archives_with_date_based_matching_calls_borg_with_date_based_f
         ),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory=None,
         borg_local_path='borg',
         borg_exit_codes=None,
@@ -605,7 +605,7 @@ def test_transfer_archives_calls_borg_with_working_directory():
         ('borg', 'transfer', '--repo', 'repo'),
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
-        extra_environment=None,
+        environment=None,
         working_directory='/working/dir',
         borg_local_path='borg',
         borg_exit_codes=None,

tests/unit/borg/test_version.py

@@ -23,7 +23,7 @@ def insert_execute_command_and_capture_output_mock(
     )
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         command,
-        extra_environment=None,
+        environment=None,
         working_directory=working_directory,
         borg_local_path=borg_local_path,
         borg_exit_codes=borg_exit_codes,

tests/unit/hooks/credential/test_container.py

@@ -0,0 +1,74 @@
+import io
+import sys
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks.credential import container as module
+
+
+@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar')))
+def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters):
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=credential_parameters
+        )
+
+
+def test_load_credential_with_invalid_secret_name_raises():
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('this is invalid',)
+        )
+
+
+def test_load_credential_reads_named_secret_from_file():
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/run/secrets/mysecret'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/run/secrets/mysecret').and_return(credential_stream)
+
+    assert (
+        module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',))
+        == 'password'
+    )
+
+
+def test_load_credential_with_custom_secrets_directory_looks_there_for_secret_file():
+    config = {'container': {'secrets_directory': '/secrets'}}
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/secrets/mysecret'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/secrets/mysecret').and_return(credential_stream)
+
+    assert (
+        module.load_credential(
+            hook_config=config['container'], config=config, credential_parameters=('mysecret',)
+        )
+        == 'password'
+    )
+
+
+def test_load_credential_with_custom_secrets_directory_prefixes_it_with_working_directory():
+    config = {'container': {'secrets_directory': 'secrets'}, 'working_directory': '/working'}
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/working/secrets/mysecret'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/working/secrets/mysecret').and_return(
+        credential_stream
+    )
+
+    assert (
+        module.load_credential(
+            hook_config=config['container'], config=config, credential_parameters=('mysecret',)
+        )
+        == 'password'
+    )
+
+
+def test_load_credential_with_file_not_found_error_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/run/secrets/mysecret').and_raise(FileNotFoundError)
+
+    with pytest.raises(ValueError):
+        module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',))

tests/unit/hooks/credential/test_file.py

@@ -0,0 +1,68 @@
+import io
+import sys
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks.credential import file as module
+
+
+@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar')))
+def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters):
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=credential_parameters
+        )
+
+
+def test_load_credential_with_invalid_credential_name_raises():
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('this is invalid',)
+        )
+
+
+def test_load_credential_reads_named_credential_from_file():
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/credentials/mycredential'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/credentials/mycredential').and_return(
+        credential_stream
+    )
+
+    assert (
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('/credentials/mycredential',)
+        )
+        == 'password'
+    )
+
+
+def test_load_credential_reads_named_credential_from_file_using_working_directory():
+    credential_stream = io.StringIO('password')
+    credential_stream.name = '/working/credentials/mycredential'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/working/credentials/mycredential').and_return(
+        credential_stream
+    )
+
+    assert (
+        module.load_credential(
+            hook_config={},
+            config={'working_directory': '/working'},
+            credential_parameters=('credentials/mycredential',),
+        )
+        == 'password'
+    )
+
+
+def test_load_credential_with_file_not_found_error_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('/credentials/mycredential').and_raise(
+        FileNotFoundError
+    )
+
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('/credentials/mycredential',)
+        )

tests/unit/hooks/credential/test_keepassxc.py

@@ -0,0 +1,80 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks.credential import keepassxc as module
+
+
+@pytest.mark.parametrize('credential_parameters', ((), ('foo',), ('foo', 'bar', 'baz')))
+def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters):
+    flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never()
+
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=credential_parameters
+        )
+
+
+def test_load_credential_with_missing_database_raises():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never()
+
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('database.kdbx', 'mypassword')
+        )
+
+
+def test_load_credential_with_present_database_fetches_password_from_keepassxc():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).with_args(
+        (
+            'keepassxc-cli',
+            'show',
+            '--show-protected',
+            '--attributes',
+            'Password',
+            'database.kdbx',
+            'mypassword',
+        )
+    ).and_return(
+        'password'
+    ).once()
+
+    assert (
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('database.kdbx', 'mypassword')
+        )
+        == 'password'
+    )
+
+
+def test_load_credential_with_custom_keepassxc_cli_command_calls_it():
+    config = {'keepassxc': {'keepassxc_cli_command': '/usr/local/bin/keepassxc-cli --some-option'}}
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).with_args(
+        (
+            '/usr/local/bin/keepassxc-cli',
+            '--some-option',
+            'show',
+            '--show-protected',
+            '--attributes',
+            'Password',
+            'database.kdbx',
+            'mypassword',
+        )
+    ).and_return(
+        'password'
+    ).once()
+
+    assert (
+        module.load_credential(
+            hook_config=config['keepassxc'],
+            config=config,
+            credential_parameters=('database.kdbx', 'mypassword'),
+        )
+        == 'password'
+    )

tests/unit/hooks/credential/test_parse.py

@@ -4,39 +4,82 @@ from flexmock import flexmock
 from borgmatic.hooks.credential import parse as module
 
 
-def test_resolve_credential_passes_through_string_without_credential_tag():
+def test_hash_adapter_is_always_equal():
+    assert module.Hash_adapter({1: 2}) == module.Hash_adapter({3: 4})
+
+
+def test_hash_adapter_alwaysh_hashes_the_same():
+    assert hash(module.Hash_adapter({1: 2})) == hash(module.Hash_adapter({3: 4}))
+
+
+def test_cache_ignoring_unhashable_arguments_caches_arguments_after_first_call():
+    hashable = 3
+    unhashable = {1, 2}
+    calls = 0
+
+    @module.cache_ignoring_unhashable_arguments
+    def function(first, second, third):
+        nonlocal calls
+        calls += 1
+
+        assert first == hashable
+        assert second == unhashable
+        assert third == unhashable
+
+        return first
+
+    assert function(hashable, unhashable, third=unhashable) == hashable
+    assert calls == 1
+
+    assert function(hashable, unhashable, third=unhashable) == hashable
+    assert calls == 1
+
+
+def test_resolve_credential_passes_through_string_without_credential():
     module.resolve_credential.cache_clear()
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never()
 
-    assert module.resolve_credential('{no credentials here}') == '{no credentials here}'
+    assert module.resolve_credential('{no credentials here}', config={}) == '{no credentials here}'
 
 
 def test_resolve_credential_passes_through_none():
     module.resolve_credential.cache_clear()
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never()
 
-    assert module.resolve_credential(None) is None
+    assert module.resolve_credential(None, config={}) is None
 
 
 @pytest.mark.parametrize('invalid_value', ('{credential}', '{credential }', '{credential systemd}'))
-def test_resolve_credential_with_invalid_credential_tag_raises(invalid_value):
+def test_resolve_credential_with_invalid_credential_raises(invalid_value):
     module.resolve_credential.cache_clear()
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never()
 
     with pytest.raises(ValueError):
-        module.resolve_credential(invalid_value)
+        module.resolve_credential(invalid_value, config={})
 
 
-def test_resolve_credential_with_valid_credential_tag_loads_credential():
+def test_resolve_credential_with_valid_credential_loads_credential():
     module.resolve_credential.cache_clear()
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args(
         'load_credential',
         {},
         'systemd',
-        'mycredential',
+        ('mycredential',),
     ).and_return('result').once()
 
-    assert module.resolve_credential('{credential systemd mycredential}') == 'result'
+    assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result'
+
+
+def test_resolve_credential_with_valid_credential_and_quoted_parameters_loads_credential():
+    module.resolve_credential.cache_clear()
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args(
+        'load_credential',
+        {},
+        'systemd',
+        ('my credential',),
+    ).and_return('result').once()
+
+    assert module.resolve_credential('{credential systemd "my credential"}', config={}) == 'result'
 
 
 def test_resolve_credential_caches_credential_after_first_call():
@@ -45,8 +88,8 @@ def test_resolve_credential_caches_credential_after_first_call():
         'load_credential',
         {},
         'systemd',
-        'mycredential',
+        ('mycredential',),
     ).and_return('result').once()
 
-    assert module.resolve_credential('{credential systemd mycredential}') == 'result'
-    assert module.resolve_credential('{credential systemd mycredential}') == 'result'
+    assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result'
+    assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result'

tests/unit/hooks/credential/test_systemd.py

@@ -7,13 +7,23 @@ from flexmock import flexmock
 from borgmatic.hooks.credential import systemd as module
 
 
+@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar')))
+def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters):
+    flexmock(module.os.environ).should_receive('get').never()
+
+    with pytest.raises(ValueError):
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=credential_parameters
+        )
+
+
 def test_load_credential_without_credentials_directory_raises():
     flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
         None
     )
 
     with pytest.raises(ValueError):
-        module.load_credential(hook_config={}, config={}, credential_name='mycredential')
+        module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',))
 
 
 def test_load_credential_with_invalid_credential_name_raises():
@@ -22,7 +32,9 @@ def test_load_credential_with_invalid_credential_name_raises():
     )
 
     with pytest.raises(ValueError):
-        module.load_credential(hook_config={}, config={}, credential_name='../../my!@#$credential')
+        module.load_credential(
+            hook_config={}, config={}, credential_parameters=('../../my!@#$credential',)
+        )
 
 
 def test_load_credential_reads_named_credential_from_file():
@@ -35,7 +47,7 @@ def test_load_credential_reads_named_credential_from_file():
     builtins.should_receive('open').with_args('/var/mycredential').and_return(credential_stream)
 
     assert (
-        module.load_credential(hook_config={}, config={}, credential_name='mycredential')
+        module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',))
         == 'password'
     )
 
@@ -48,4 +60,4 @@ def test_load_credential_with_file_not_found_error_raises():
     builtins.should_receive('open').with_args('/var/mycredential').and_raise(FileNotFoundError)
 
     with pytest.raises(ValueError):
-        module.load_credential(hook_config={}, config={}, credential_name='mycredential')
+        module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',))

tests/unit/hooks/data_source/test_btrfs.py

@@ -1,7 +1,7 @@
 import pytest
 from flexmock import flexmock
 
-from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
+from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
 from borgmatic.hooks.data_source import btrfs as module
 
 
@@ -52,9 +52,14 @@ def test_get_subvolume_mount_points_with_findmnt_json_missing_filesystems_errors
 def test_get_subvolumes_collects_subvolumes_matching_patterns():
     flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2'))
 
+    contained_pattern = Pattern(
+        '/mnt1',
+        type=Pattern_type.ROOT,
+        source=Pattern_source.CONFIG,
+    )
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
         'get_contained_patterns'
-    ).with_args('/mnt1', object).and_return((Pattern('/mnt1'),))
+    ).with_args('/mnt1', object).and_return((contained_pattern,))
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
         'get_contained_patterns'
     ).with_args('/mnt2', object).and_return(())
@@ -66,7 +71,69 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns():
             Pattern('/mnt1'),
             Pattern('/mnt3'),
         ],
-    ) == (module.Subvolume('/mnt1', contained_patterns=(Pattern('/mnt1'),)),)
+    ) == (module.Subvolume('/mnt1', contained_patterns=(contained_pattern,)),)
+
+
+def test_get_subvolumes_skips_non_root_patterns():
+    flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2'))
+
+    flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
+        'get_contained_patterns'
+    ).with_args('/mnt1', object).and_return(
+        (
+            Pattern(
+                '/mnt1',
+                type=Pattern_type.EXCLUDE,
+                source=Pattern_source.CONFIG,
+            ),
+        )
+    )
+    flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
+        'get_contained_patterns'
+    ).with_args('/mnt2', object).and_return(())
+
+    assert (
+        module.get_subvolumes(
+            'btrfs',
+            'findmnt',
+            patterns=[
+                Pattern('/mnt1'),
+                Pattern('/mnt3'),
+            ],
+        )
+        == ()
+    )
+
+
+def test_get_subvolumes_skips_non_config_patterns():
+    flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2'))
+
+    flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
+        'get_contained_patterns'
+    ).with_args('/mnt1', object).and_return(
+        (
+            Pattern(
+                '/mnt1',
+                type=Pattern_type.ROOT,
+                source=Pattern_source.HOOK,
+            ),
+        )
+    )
+    flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
+        'get_contained_patterns'
+    ).with_args('/mnt2', object).and_return(())
+
+    assert (
+        module.get_subvolumes(
+            'btrfs',
+            'findmnt',
+            patterns=[
+                Pattern('/mnt1'),
+                Pattern('/mnt3'),
+            ],
+        )
+        == ()
+    )
 
 
 def test_get_subvolumes_without_patterns_collects_all_subvolumes():
@@ -520,6 +587,18 @@ def test_remove_data_source_dumps_deletes_snapshots():
     flexmock(module).should_receive('delete_snapshot').with_args(
         'btrfs', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2'
     ).never()
+    flexmock(module.os.path).should_receive('isdir').with_args(
+        '/mnt/subvol1/.borgmatic-1234'
+    ).and_return(True)
+    flexmock(module.os.path).should_receive('isdir').with_args(
+        '/mnt/subvol1/.borgmatic-5678'
+    ).and_return(True)
+    flexmock(module.os.path).should_receive('isdir').with_args(
+        '/mnt/subvol2/.borgmatic-1234'
+    ).and_return(True)
+    flexmock(module.os.path).should_receive('isdir').with_args(
+        '/mnt/subvol2/.borgmatic-5678'
+    ).and_return(True)
     flexmock(module.shutil).should_receive('rmtree').with_args(
         '/mnt/subvol1/.borgmatic-1234'
     ).once()
@@ -846,3 +925,48 @@ def test_remove_data_source_dumps_with_delete_snapshot_called_process_error_bail
         borgmatic_runtime_directory='/run/borgmatic',
         dry_run=False,
     )
+
+
+def test_remove_data_source_dumps_with_root_subvolume_skips_duplicate_removal():
+    config = {'btrfs': {}}
+    flexmock(module).should_receive('get_subvolumes').and_return(
+        (module.Subvolume('/', contained_patterns=(Pattern('/etc'),)),)
+    )
+
+    flexmock(module).should_receive('make_snapshot_path').with_args('/').and_return(
+        '/.borgmatic-1234'
+    )
+
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'replace_temporary_subdirectory_with_glob'
+    ).with_args(
+        '/.borgmatic-1234',
+        temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX,
+    ).and_return(
+        '/.borgmatic-*'
+    )
+
+    flexmock(module.glob).should_receive('glob').with_args('/.borgmatic-*').and_return(
+        ('/.borgmatic-1234', '/.borgmatic-5678')
+    )
+
+    flexmock(module.os.path).should_receive('isdir').with_args('/.borgmatic-1234').and_return(
+        True
+    ).and_return(False)
+    flexmock(module.os.path).should_receive('isdir').with_args('/.borgmatic-5678').and_return(
+        True
+    ).and_return(False)
+
+    flexmock(module).should_receive('delete_snapshot').with_args('btrfs', '/.borgmatic-1234').once()
+    flexmock(module).should_receive('delete_snapshot').with_args('btrfs', '/.borgmatic-5678').once()
+
+    flexmock(module.os.path).should_receive('isdir').with_args('').and_return(False)
+
+    flexmock(module.shutil).should_receive('rmtree').never()
+
+    module.remove_data_source_dumps(
+        hook_config=config['btrfs'],
+        config=config,
+        borgmatic_runtime_directory='/run/borgmatic',
+        dry_run=False,
+    )