Add a passphrase hook so borgmatic can collect the encryption passphrase once and pass it to Borg multiple times (#961).

borgmatic/borg/environment.py

@@ -1,5 +1,7 @@
 import os
 
+import borgmatic.hooks.dispatch
+
 OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_base_directory': 'BORG_BASE_DIR',
     'borg_config_directory': 'BORG_CONFIG_DIR',
@@ -7,7 +9,6 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
     'borg_security_directory': 'BORG_SECURITY_DIR',
     'borg_keys_directory': 'BORG_KEYS_DIR',
-    'encryption_passcommand': 'BORG_PASSCOMMAND',
     'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
     'temporary_directory': 'TMPDIR',
@@ -36,6 +37,24 @@ def make_environment(config):
         if value:
             environment[environment_variable_name] = str(value)
 
+    passphrase = borgmatic.hooks.dispatch.call_hook(
+        function_name='load_credential',
+        config=config,
+        hook_name='passcommand',
+        credential_name='encryption_passphrase',
+    )
+
+    # If the passcommand produced a passphrase, send it to Borg via an anonymous pipe.
+    if passphrase:
+        read_file_descriptor, write_file_descriptor = os.pipe()
+        os.write(write_file_descriptor, passphrase.encode('utf-8'))
+        os.close(write_file_descriptor)
+
+        # This, plus subprocess.Popen(..., close_fds=False) in execute.py, is necessary for the Borg
+        # child process to inherit the file descriptor.
+        os.set_inheritable(read_file_descriptor, True)
+        environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor)
+
     for (
         option_name,
         environment_variable_name,

borgmatic/execute.py

@@ -307,6 +307,8 @@ def execute_command(
         shell=shell,
         env=environment,
         cwd=working_directory,
+        # Necessary for the passcommand credential hook to work.
+        close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
     )
     if not run_to_completion:
         return process
@@ -354,6 +356,8 @@ def execute_command_and_capture_output(
             shell=shell,
             env=environment,
             cwd=working_directory,
+            # Necessary for the passcommand credential hook to work.
+            close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
         )
     except subprocess.CalledProcessError as error:
         if (
@@ -414,6 +418,8 @@ def execute_command_with_processes(
             shell=shell,
             env=environment,
             cwd=working_directory,
+            # Necessary for the passcommand credential hook to work.
+            close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')),
         )
     except (subprocess.CalledProcessError, OSError):
         # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.

borgmatic/hooks/credential/passcommand.py

@@ -0,0 +1,61 @@
+import functools
+import logging
+import shlex
+
+
+import borgmatic.config.paths
+import borgmatic.execute
+
+
+logger = logging.getLogger(__name__)
+
+
+@functools.cache
+def run_passcommand(passcommand, passphrase_configured, working_directory):
+    '''
+    Run the given passcommand using the given working directory and return the passphrase produced
+    by the command. But bail first if a passphrase is already configured; this mimics Borg's
+    behavior.
+
+    Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
+    per borgmatic invocation.
+    '''
+    if passcommand and passphrase_configured:
+        logger.warning(
+            'Ignoring the "encryption_passcommand" option because "encryption_passphrase" is set'
+        )
+        return None
+
+    return borgmatic.execute.execute_command_and_capture_output(
+        shlex.split(passcommand),
+        working_directory=working_directory,
+    )
+
+
+def load_credential(hook_config, config, credential_name):
+    '''
+    Given the hook configuration dict, the configuration dict, and a credential name to load, call
+    the configured passcommand to produce and return an encryption passphrase. In effect, we're
+    doing an end-run around Borg by invoking its passcommand ourself. This allows us to pass the
+    resulting passphrase to multiple different Borg invocations without the user having to be
+    prompted multiple times.
+
+    If no passcommand is configured, then return None.
+
+    The credential name must be "encryption_passphrase"; that's the only supported credential with
+    this particular hook.
+    '''
+    if credential_name != 'encryption_passphrase':
+        raise ValueError(
+            f'Credential name "{credential_name}" is not supported for the passphrase credential hook'
+        )
+
+    passcommand = config.get('encryption_passcommand')
+
+    if not passcommand:
+        return None
+
+    passphrase = config.get('encryption_passphrase')
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
+    return run_passcommand(passcommand, bool(passphrase), working_directory)

borgmatic/hooks/dispatch.py

@@ -3,6 +3,7 @@ import importlib
 import logging
 import pkgutil
 
+import borgmatic.hooks.credential
 import borgmatic.hooks.data_source
 import borgmatic.hooks.monitoring
 
@@ -10,6 +11,7 @@ logger = logging.getLogger(__name__)
 
 
 class Hook_type(enum.Enum):
+    CREDENTIAL = 'credential'
     DATA_SOURCE = 'data_source'
     MONITORING = 'monitoring'
 
@@ -40,7 +42,7 @@ def call_hook(function_name, config, hook_name, *args, **kwargs):
     module_name = hook_name.split('_databases')[0]
 
     # Probe for a data source or monitoring hook module corresponding to the hook name.
-    for parent_module in (borgmatic.hooks.data_source, borgmatic.hooks.monitoring):
+    for parent_module in (borgmatic.hooks.credential, borgmatic.hooks.data_source, borgmatic.hooks.monitoring):
         if module_name not in get_submodule_names(parent_module):
             continue
 
@@ -62,8 +64,8 @@ def call_hook(function_name, config, hook_name, *args, **kwargs):
 def call_hooks(function_name, config, hook_type, *args, **kwargs):
     '''
     Given a configuration dict, call the requested function of the Python module corresponding to
-    each hook of the given hook type (either "data_source" or "monitoring"). Supply each call with
-    the configuration for that hook, and any given args and kwargs.
+    each hook of the given hook type ("credential", "data_source", or "monitoring"). Supply each
+    call with the configuration for that hook, and any given args and kwargs.
 
     Collect any return values into a dict from module name to return value. Note that the module
     name is the name of the hook module itself, which might be different from the hook configuration
@@ -88,9 +90,9 @@ def call_hooks(function_name, config, hook_type, *args, **kwargs):
 def call_hooks_even_if_unconfigured(function_name, config, hook_type, *args, **kwargs):
     '''
     Given a configuration dict, call the requested function of the Python module corresponding to
-    each hook of the given hook type (either "data_source" or "monitoring"). Supply each call with
-    the configuration for that hook and any given args and kwargs. Collect any return values into a
-    dict from hook name to return value.
+    each hook of the given hook type ("credential", "data_source", or "monitoring"). Supply each
+    call with the configuration for that hook and any given args and kwargs. Collect any return
+    values into a dict from hook name to return value.
 
     Raise AttributeError if the function name is not found in the module.
     Raise anything else that a called function raises. An error stops calls to subsequent functions.

docs/reference/source-code.md

@@ -18,6 +18,7 @@ you get started. Starting at the top level, we have:
    * [commands](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/commands): Looking to add a new flag or action? Start here. This contains borgmatic's entry point, argument parsing, and shell completion. 
    * [config](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/config): Code responsible for loading, normalizing, and validating borgmatic's configuration. Interested in adding a new configuration option? Check out `schema.yaml` here.
    * [hooks](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks): Looking to add a new database, filesystem, or monitoring integration? Start here.
+     * [credential](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks/credential): Credential hooks—for loading passphrases and secrets from external providers.
      * [data_source](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks/data_source): Database and filesystem hooks—anything that produces data or files to go into a backup archive.
      * [monitoring](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks/monitoring): Monitoring hooks—integrations with third-party or self-hosted monitoring services.
  * [docs](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/docs): How-to and reference documentation, including the document you're reading now.