Bump version for release.

Merge pull request #77 from hop/main

.drone.yml

@@ -13,19 +13,30 @@ services:
     environment:
       POSTGRES_PASSWORD: test2
       POSTGRES_DB: test
-      POSTGRES_USER: postgres2
     commands:
       - docker-entrypoint.sh -p 5433
-  - name: mysql
-    image: docker.io/mariadb:10.5
+  - name: mariadb
+    image: docker.io/mariadb:10.11.4
     environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-  - name: mysql2
-    image: docker.io/mariadb:10.5
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: test
+  - name: mariadb2
+    image: docker.io/mariadb:10.11.4
     environment:
-      MYSQL_ROOT_PASSWORD: test2
-      MYSQL_DATABASE: test
+      MARIADB_ROOT_PASSWORD: test2
+      MARIADB_DATABASE: test
+    commands:
+      - docker-entrypoint.sh --port=3307
+  - name: not-actually-mysql
+    image: docker.io/mariadb:10.11.4
+    environment:
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: test
+  - name: not-actually-mysql2
+    image: docker.io/mariadb:10.11.4
+    environment:
+      MARIADB_ROOT_PASSWORD: test2
+      MARIADB_DATABASE: test
     commands:
       - docker-entrypoint.sh --port=3307
   - name: mongodb

NEWS

@@ -1,3 +1,31 @@
+1.8.2
+ * #345: Add "key export" action to export a copy of the repository key for safekeeping in case
+   the original goes missing or gets damaged.
+ * #727: Add a MariaDB database hook that uses native MariaDB commands instead of the deprecated
+   MySQL ones. Be aware though that any existing backups made with the "mysql_databases:" hook are
+   only restorable with a "mysql_databases:" configuration.
+ * #738: Fix for potential data loss (data not getting restored) in which the database "restore"
+   action didn't actually restore anything and indicated success anyway.
+ * Remove the deprecated use of the MongoDB hook's "--db" flag for database restoration.
+ * Add source code reference documentation for getting oriented with the borgmatic code as a
+   developer: https://torsion.org/borgmatic/docs/reference/source-code/
+
+1.8.1
+ * #326: Add documentation for restoring a database to an alternate host:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-to-an-alternate-host
+ * #697: Add documentation for "bootstrap" action:
+   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
+ * #725: Add "store_config_files" option for disabling the automatic backup of configuration files
+   used by the "config bootstrap" action.
+ * #728: Fix for "prune" action error when using the "keep_exclude_tags" option.
+ * #730: Fix for Borg's interactive prompt on the "check --repair" action automatically getting
+   answered "NO" even when the "check_i_know_what_i_am_doing" option isn't set.
+ * #732: Include multiple configuration files with a single "!include". See the documentation for
+   more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#multiple-merge-includes
+ * #734: Omit "--glob-archives" or "--match-archives" Borg flag when its value would be "*" (meaning
+   all archives).
+
 1.8.0
  * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting
    repository/archive into the resulting Borg command-line, pass repository to Borg via an

borgmatic/actions/create.py

@@ -91,7 +91,10 @@ def run_create(
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         global_arguments.dry_run,
     )
-    create_borgmatic_manifest(config, global_arguments.used_config_paths, global_arguments.dry_run)
+    if config.get('store_config_files', True):
+        create_borgmatic_manifest(
+            config, global_arguments.used_config_paths, global_arguments.dry_run
+        )
     stream_processes = [process for processes in active_dumps.values() for process in processes]
 
     json_output = borgmatic.borg.create.create_archive(

borgmatic/actions/export_key.py

@@ -0,0 +1,33 @@
+import logging
+
+import borgmatic.borg.export_key
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_export_key(
+    repository,
+    config,
+    local_borg_version,
+    export_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "key export" action for the given repository.
+    '''
+    if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, export_arguments.repository
+    ):
+        logger.info(f'{repository.get("label", repository["path"])}: Exporting repository key')
+        borgmatic.borg.export_key.export_key(
+            repository['path'],
+            config,
+            local_borg_version,
+            export_arguments,
+            global_arguments,
+            local_path=local_path,
+            remote_path=remote_path,
+        )

borgmatic/actions/restore.py

@@ -27,7 +27,8 @@ def get_configured_database(
     hooks for the named database. If a configuration database name is given, use that instead of the
     database name to lookup the database in the given hooks configuration.
 
-    Return the found database as a tuple of (found hook name, database configuration dict).
+    Return the found database as a tuple of (found hook name, database configuration dict) or (None,
+    None) if not found.
     '''
     if not configuration_database_name:
         configuration_database_name = database_name
@@ -39,7 +40,10 @@ def get_configured_database(
             if hook_name in borgmatic.hooks.dump.DATABASE_HOOK_NAMES
         }
     else:
-        hooks_to_search = {hook_name: config[hook_name]}
+        try:
+            hooks_to_search = {hook_name: config[hook_name]}
+        except KeyError:
+            return (None, None)
 
     return next(
         (
@@ -73,9 +77,9 @@ def restore_single_database(
     connection_params,
 ):  # pragma: no cover
     '''
-    Given (among other things) an archive name, a database hook name, the hostname,
-    port, username and password as connection params, and a configured database
-    configuration dict, restore that database from the archive.
+    Given (among other things) an archive name, a database hook name, the hostname, port,
+    username/password as connection params, and a configured database configuration dict, restore
+    that database from the archive.
     '''
     logger.info(
         f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}'
@@ -108,14 +112,14 @@ def restore_single_database(
 
     # Run a single database restore, consuming the extract stdout (if any).
     borgmatic.hooks.dispatch.call_hooks(
-        'restore_database_dump',
-        config,
-        repository['path'],
-        database['name'],
-        borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
-        global_arguments.dry_run,
-        extract_process,
-        connection_params,
+        function_name='restore_database_dump',
+        config=config,
+        log_prefix=repository['path'],
+        hook_names=[hook_name],
+        database=database,
+        dry_run=global_arguments.dry_run,
+        extract_process=extract_process,
+        connection_params=connection_params,
     )
 
 
@@ -333,7 +337,7 @@ def run_restore(
                 connection_params,
             )
 
-    # For any database that weren't found via exact matches in the configuration, try to fallback
+    # For any databases that weren't found via exact matches in the configuration, try to fallback
     # to "all" entries.
     for hook_name, database_names in remaining_restore_names.items():
         for database_name in database_names:

borgmatic/borg/check.py

@@ -142,7 +142,7 @@ def filter_checks_on_frequency(
         if datetime.datetime.now() < check_time + frequency_delta:
             remaining = check_time + frequency_delta - datetime.datetime.now()
             logger.info(
-                f'Skipping {check} check due to configured frequency; {remaining} until next check'
+                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
             )
             filtered_checks.remove(check)
 

borgmatic/borg/create.py

@@ -354,7 +354,11 @@ def create_archive(
             expand_directories(
                 tuple(config.get('source_directories', ()))
                 + borgmatic_source_directories
-                + tuple(global_arguments.used_config_paths)
+                + tuple(
+                    global_arguments.used_config_paths
+                    if config.get('store_config_files', True)
+                    else ()
+                )
             )
         ),
         additional_directory_devices=map_directories_to_devices(
@@ -416,7 +420,7 @@ def create_archive(
             f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
 
-    create_command = (
+    create_flags = (
         tuple(local_path.split(' '))
         + ('create',)
         + make_pattern_flags(config, pattern_file.name if pattern_file else None)
@@ -445,12 +449,12 @@ def create_archive(
         )
         + (('--dry-run',) if dry_run else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
-        + flags.make_repository_archive_flags(
-            repository_path, archive_name_format, local_borg_version
-        )
-        + (sources if not pattern_file else ())
     )
 
+    create_positional_arguments = flags.make_repository_archive_flags(
+        repository_path, archive_name_format, local_borg_version
+    ) + (sources if not pattern_file else ())
+
     if json:
         output_log_level = None
     elif list_files or (stats and not dry_run):
@@ -469,7 +473,7 @@ def create_archive(
     if stream_processes and not config.get('read_special'):
         logger.debug(f'{repository_path}: Collecting special file paths')
         special_file_paths = collect_special_file_paths(
-            create_command,
+            create_flags + create_positional_arguments,
             local_path,
             working_directory,
             borg_environment,
@@ -486,9 +490,9 @@ def create_archive(
                 ),
                 pattern_file=exclude_file,
             )
-            create_command += make_exclude_flags(config, exclude_file.name)
+            create_flags += make_exclude_flags(config, exclude_file.name)
 
-    create_command += (
+    create_flags += (
         (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
         + (('--stats',) if stats and not json and not dry_run else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
@@ -498,7 +502,7 @@ def create_archive(
 
     if stream_processes:
         return execute_command_with_processes(
-            create_command,
+            create_flags + create_positional_arguments,
             stream_processes,
             output_log_level,
             output_file,
@@ -508,14 +512,14 @@ def create_archive(
         )
     elif output_log_level is None:
         return execute_command_and_capture_output(
-            create_command,
+            create_flags + create_positional_arguments,
             working_directory=working_directory,
             extra_environment=borg_environment,
             borg_local_path=local_path,
         )
     else:
         execute_command(
-            create_command,
+            create_flags + create_positional_arguments,
             output_log_level,
             output_file,
             borg_local_path=local_path,

borgmatic/borg/environment.py

@@ -38,14 +38,16 @@ def make_environment(config):
         option_name,
         environment_variable_name,
     ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
-        value = config.get(option_name, False)
-        environment[environment_variable_name] = 'yes' if value else 'no'
+        value = config.get(option_name)
+        if value is not None:
+            environment[environment_variable_name] = 'yes' if value else 'no'
 
     for (
         option_name,
         environment_variable_name,
     ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items():
-        value = config.get(option_name, False)
-        environment[environment_variable_name] = 'YES' if value else 'NO'
+        value = config.get(option_name)
+        if value is not None:
+            environment[environment_variable_name] = 'YES' if value else 'NO'
 
     return environment

borgmatic/borg/export_key.py

@@ -0,0 +1,70 @@
+import logging
+import os
+
+import borgmatic.logger
+from borgmatic.borg import environment, flags
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def export_key(
+    repository_path,
+    config,
+    local_borg_version,
+    export_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, and
+    optional local and remote Borg paths, export the repository key to the destination path
+    indicated in the export arguments.
+
+    If the destination path is empty or "-", then print the key to stdout instead of to a file.
+
+    Raise FileExistsError if a path is given but it already exists on disk.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+    umask = config.get('umask', None)
+    lock_wait = config.get('lock_wait', None)
+
+    if export_arguments.path and export_arguments.path != '-':
+        if os.path.exists(export_arguments.path):
+            raise FileExistsError(
+                f'Destination path {export_arguments.path} already exists. Aborting.'
+            )
+
+        output_file = None
+    else:
+        output_file = DO_NOT_CAPTURE
+
+    full_command = (
+        (local_path, 'key', 'export')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + flags.make_flags('paper', export_arguments.paper)
+        + flags.make_flags('qr-html', export_arguments.qr_html)
+        + flags.make_repository_flags(
+            repository_path,
+            local_borg_version,
+        )
+        + ((export_arguments.path,) if output_file is None else ())
+    )
+
+    if global_arguments.dry_run:
+        logging.info(f'{repository_path}: Skipping key export (dry run)')
+        return
+
+    execute_command(
+        full_command,
+        output_file=output_file,
+        output_log_level=logging.ANSWER,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(config),
+    )

borgmatic/borg/flags.py

@@ -77,6 +77,9 @@ def make_match_archives_flags(match_archives, archive_name_format, local_borg_ve
 
     derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
 
+    if derived_match_archives == '*':
+        return ()
+
     if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
         return ('--match-archives', f'sh:{derived_match_archives}')
     else:

borgmatic/borg/prune.py

@@ -26,7 +26,7 @@ def make_prune_flags(config, local_borg_version):
     flag_pairs = (
         ('--' + option_name.replace('_', '-'), str(value))
         for option_name, value in config.items()
-        if option_name.startswith('keep_')
+        if option_name.startswith('keep_') and option_name != 'keep_exclude_tags'
     )
     prefix = config.get('prefix')
 

borgmatic/commands/arguments.py

@@ -23,6 +23,7 @@ ACTION_ALIASES = {
     'info': ['-i'],
     'transfer': [],
     'break-lock': [],
+    'key': [],
     'borg': [],
 }
 
@@ -331,9 +332,8 @@ def make_parsers():
 
     global_plus_action_parser = ArgumentParser(
         description='''
-            Simple, configuration-driven backup software for servers and workstations. If none of
-            the action options are given, then borgmatic defaults to: create, prune, compact, and
-            check.
+            Simple, configuration-driven backup software for servers and workstations. If no actions
+            are given, then borgmatic defaults to: create, prune, compact, and check.
             ''',
         parents=[global_parser],
     )
@@ -1177,6 +1177,51 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
+    key_parser = action_parsers.add_parser(
+        'key',
+        aliases=ACTION_ALIASES['key'],
+        help='Perform repository key related operations',
+        description='Perform repository key related operations',
+        add_help=False,
+    )
+
+    key_group = key_parser.add_argument_group('key arguments')
+    key_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    key_parsers = key_parser.add_subparsers(
+        title='key sub-actions',
+    )
+
+    key_export_parser = key_parsers.add_parser(
+        'export',
+        help='Export a copy of the repository key for safekeeping in case the original goes missing or gets damaged',
+        description='Export a copy of the repository key for safekeeping in case the original goes missing or gets damaged',
+        add_help=False,
+    )
+    key_export_group = key_export_parser.add_argument_group('key export arguments')
+    key_export_group.add_argument(
+        '--paper',
+        action='store_true',
+        help='Export the key in a text format suitable for printing and later manual entry',
+    )
+    key_export_group.add_argument(
+        '--qr-html',
+        action='store_true',
+        help='Export the key in an HTML format suitable for printing and later manual entry or QR code scanning',
+    )
+    key_export_group.add_argument(
+        '--repository',
+        help='Path of repository to export the key for, defaults to the configured repository if there is only one',
+    )
+    key_export_group.add_argument(
+        '--path',
+        metavar='PATH',
+        help='Path to export the key to, defaults to stdout (but be careful about dirtying the output with --verbosity)',
+    )
+    key_export_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
     borg_parser = action_parsers.add_parser(
         'borg',
         aliases=ACTION_ALIASES['borg'],

borgmatic/commands/borgmatic.py

@@ -22,6 +22,7 @@ import borgmatic.actions.config.bootstrap
 import borgmatic.actions.config.generate
 import borgmatic.actions.config.validate
 import borgmatic.actions.create
+import borgmatic.actions.export_key
 import borgmatic.actions.export_tar
 import borgmatic.actions.extract
 import borgmatic.actions.info
@@ -448,6 +449,16 @@ def run_actions(
                 local_path,
                 remote_path,
             )
+        elif action_name == 'export':
+            borgmatic.actions.export_key.run_export_key(
+                repository,
+                config,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
         elif action_name == 'borg':
             borgmatic.actions.borg.run_borg(
                 repository,

borgmatic/config/load.py

@@ -1,6 +1,8 @@
 import functools
+import itertools
 import json
 import logging
+import operator
 import os
 
 import ruamel.yaml
@@ -8,34 +10,67 @@ import ruamel.yaml
 logger = logging.getLogger(__name__)
 
 
+def probe_and_include_file(filename, include_directories):
+    '''
+    Given a filename to include and a list of include directories to search for matching files,
+    probe for the file, load it, and return the loaded configuration as a data structure of nested
+    dicts, lists, etc.
+
+    Raise FileNotFoundError if the included file was not found.
+    '''
+    expanded_filename = os.path.expanduser(filename)
+
+    if os.path.isabs(expanded_filename):
+        return load_configuration(expanded_filename)
+
+    candidate_filenames = {
+        os.path.join(directory, expanded_filename) for directory in include_directories
+    }
+
+    for candidate_filename in candidate_filenames:
+        if os.path.exists(candidate_filename):
+            return load_configuration(candidate_filename)
+
+    raise FileNotFoundError(
+        f'Could not find include {filename} at {" or ".join(candidate_filenames)}'
+    )
+
+
 def include_configuration(loader, filename_node, include_directory):
     '''
-    Given a ruamel.yaml.loader.Loader, a ruamel.yaml.serializer.ScalarNode containing the included
-    filename, and an include directory path to search for matching files, load the given YAML
-    filename (ignoring the given loader so we can use our own) and return its contents as a data
-    structure of nested dicts and lists. If the filename is relative, probe for it within 1. the
-    current working directory and 2. the given include directory.
+    Given a ruamel.yaml.loader.Loader, a ruamel.yaml.nodes.ScalarNode containing the included
+    filename (or a list containing multiple such filenames), and an include directory path to search
+    for matching files, load the given YAML filenames (ignoring the given loader so we can use our
+    own) and return their contents as data structure of nested dicts, lists, etc. If the given
+    filename node's value is a scalar string, then the return value will be a single value. But if
+    the given node value is a list, then the return value will be a list of values, one per loaded
+    configuration file.
+
+    If a filename is relative, probe for it within 1. the current working directory and 2. the given
+    include directory.
 
     Raise FileNotFoundError if an included file was not found.
     '''
     include_directories = [os.getcwd(), os.path.abspath(include_directory)]
-    include_filename = os.path.expanduser(filename_node.value)
 
-    if not os.path.isabs(include_filename):
-        candidate_filenames = [
-            os.path.join(directory, include_filename) for directory in include_directories
+    if isinstance(filename_node.value, str):
+        return probe_and_include_file(filename_node.value, include_directories)
+
+    if (
+        isinstance(filename_node.value, list)
+        and len(filename_node.value)
+        and isinstance(filename_node.value[0], ruamel.yaml.nodes.ScalarNode)
+    ):
+        # Reversing the values ensures the correct ordering if these includes are subsequently
+        # merged together.
+        return [
+            probe_and_include_file(node.value, include_directories)
+            for node in reversed(filename_node.value)
         ]
 
-        for candidate_filename in candidate_filenames:
-            if os.path.exists(candidate_filename):
-                include_filename = candidate_filename
-                break
-        else:
-            raise FileNotFoundError(
-                f'Could not find include {filename_node.value} at {" or ".join(candidate_filenames)}'
-            )
-
-    return load_configuration(include_filename)
+    raise ValueError(
+        '!include value is not supported; use a single filename or a list of filenames'
+    )
 
 
 def raise_retain_node_error(loader, node):
@@ -53,7 +88,7 @@ def raise_retain_node_error(loader, node):
             'The !retain tag may only be used within a configuration file containing a merged !include tag.'
         )
 
-    raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
+    raise ValueError('The !retain tag may only be used on a mapping or list.')
 
 
 def raise_omit_node_error(loader, node):
@@ -65,14 +100,14 @@ def raise_omit_node_error(loader, node):
     tags are handled by deep_merge_nodes() below.
     '''
     raise ValueError(
-        'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.'
+        'The !omit tag may only be used on a scalar (e.g., string) or list element within a configuration file containing a merged !include tag.'
     )
 
 
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
-    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    separate YAML configuration files. Example syntax: `option: !include common.yaml`
     '''
 
     def __init__(self, preserve_quotes=None, loader=None, include_directory=None):
@@ -81,6 +116,9 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
             '!include',
             functools.partial(include_configuration, include_directory=include_directory),
         )
+
+        # These are catch-all error handlers for tags that don't get applied and removed by
+        # deep_merge_nodes() below.
         self.add_constructor('!retain', raise_retain_node_error)
         self.add_constructor('!omit', raise_omit_node_error)
 
@@ -90,8 +128,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
         using the YAML '<<' merge key. Example syntax:
 
         ```
-        retention:
-            keep_daily: 1
+        option:
+            sub_option: 1
 
         <<: !include common.yaml
         ```
@@ -104,9 +142,15 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
 
         for index, (key_node, value_node) in enumerate(node.value):
             if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
-                included_value = representer.represent_data(self.construct_object(value_node))
-                node.value[index] = (key_node, included_value)
+                # Replace the merge include with a sequence of included configuration nodes ready
+                # for merging. The construct_object() call here triggers include_configuration()
+                # among other constructors.
+                node.value[index] = (
+                    key_node,
+                    representer.represent_data(self.construct_object(value_node)),
+                )
 
+        # This super().flatten_mapping() call actually performs "<<" merges.
         super(Include_constructor, self).flatten_mapping(node)
 
         node.value = deep_merge_nodes(node.value)
@@ -138,7 +182,12 @@ def load_configuration(filename):
         file_contents = file.read()
         config = yaml.load(file_contents)
 
-        if config and 'constants' in config:
+        try:
+            has_constants = bool(config and 'constants' in config)
+        except TypeError:
+            has_constants = False
+
+        if has_constants:
             for key, value in config['constants'].items():
                 value = json.dumps(value)
                 file_contents = file_contents.replace(f'{{{key}}}', value.strip('"'))
@@ -149,53 +198,92 @@ def load_configuration(filename):
         return config
 
 
-def filter_omitted_nodes(nodes):
+def filter_omitted_nodes(nodes, values):
     '''
-    Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a
-    value matching such nodes.
+    Given a nested borgmatic configuration data structure as a list of tuples in the form of:
+
+    [
+        (
+            ruamel.yaml.nodes.ScalarNode as a key,
+            ruamel.yaml.nodes.MappingNode or other Node as a value,
+        ),
+        ...
+    ]
+
+    ... and a combined list of all values for those nodes, return a filtered list of the values,
+    omitting any that have an "!omit" tag (or with a value matching such nodes).
+
+    But if only a single node is given, bail and return the given values unfiltered, as "!omit" only
+    applies when there are merge includes (and therefore multiple nodes).
     '''
-    omitted_values = tuple(node.value for node in nodes if node.tag == '!omit')
+    if len(nodes) <= 1:
+        return values
 
-    return [node for node in nodes if node.value not in omitted_values]
+    omitted_values = tuple(node.value for node in values if node.tag == '!omit')
+
+    return [node for node in values if node.value not in omitted_values]
 
 
-DELETED_NODE = object()
+def merge_values(nodes):
+    '''
+    Given a nested borgmatic configuration data structure as a list of tuples in the form of:
+
+    [
+        (
+            ruamel.yaml.nodes.ScalarNode as a key,
+            ruamel.yaml.nodes.MappingNode or other Node as a value,
+        ),
+        ...
+    ]
+
+    ... merge its sequence or mapping node values and return the result. For sequence nodes, this
+    means appending together its contained lists. For mapping nodes, it means merging its contained
+    dicts.
+    '''
+    return functools.reduce(operator.add, (value.value for key, value in nodes))
 
 
 def deep_merge_nodes(nodes):
     '''
     Given a nested borgmatic configuration data structure as a list of tuples in the form of:
 
+    [
         (
             ruamel.yaml.nodes.ScalarNode as a key,
             ruamel.yaml.nodes.MappingNode or other Node as a value,
         ),
+        ...
+    ]
 
-    ... deep merge any node values corresponding to duplicate keys and return the result. If
-    there are colliding keys with non-MappingNode values (e.g., integers or strings), the last
-    of the values wins.
+    ... deep merge any node values corresponding to duplicate keys and return the result. The
+    purpose of merging like this is to support, for instance, merging one borgmatic configuration
+    file into another for reuse, such that a configuration option with sub-options does not
+    completely replace the corresponding option in a merged file.
+
+    If there are colliding keys with scalar values (e.g., integers or strings), the last of the
+    values wins.
 
     For instance, given node values of:
 
         [
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option1'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='1')
                     ),
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='7')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='2')
                     ),
                 ]),
             ),
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
                         ScalarNode(tag='tag:yaml.org,2002:int', value='5')
                     ),
                 ]),
@@ -206,88 +294,95 @@ def deep_merge_nodes(nodes):
 
         [
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option1'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='1')
                     ),
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
                         ScalarNode(tag='tag:yaml.org,2002:int', value='5')
                     ),
                 ]),
             ),
         ]
 
+    This function supports multi-way merging, meaning that if the same option name exists three or
+    more times (at the same scope level), all of those instances get merged together.
+
     If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged.
 
-    The purpose of deep merging like this is to support, for instance, merging one borgmatic
-    configuration file into another for reuse, such that a configuration option with sub-options
-    does not completely replace the corresponding option in a merged file.
-
-    Raise ValueError if a merge is implied using two incompatible types.
+    Raise ValueError if a merge is implied using multiple incompatible types.
     '''
-    # Map from original node key/value to the replacement merged node. DELETED_NODE as a replacement
-    # node indications deletion.
-    replaced_nodes = {}
+    merged_nodes = []
 
-    # To find nodes that require merging, compare each node with each other node.
-    for a_key, a_value in nodes:
-        for b_key, b_value in nodes:
-            # If we've already considered one of the nodes for merging, skip it.
-            if (a_key, a_value) in replaced_nodes or (b_key, b_value) in replaced_nodes:
-                continue
+    def get_node_key_name(node):
+        return node[0].value
 
-            # If the keys match and the values are different, we need to merge these two A and B nodes.
-            if a_key.tag == b_key.tag and a_key.value == b_key.value and a_value != b_value:
-                if not type(a_value) is type(b_value):
-                    raise ValueError(
-                        f'Incompatible types found when trying to merge "{a_key.value}:" values across configuration files: {type(a_value).id} and {type(b_value).id}'
+    # Bucket the nodes by their keys. Then merge all of the values sharing the same key.
+    for key_name, grouped_nodes in itertools.groupby(
+        sorted(nodes, key=get_node_key_name), get_node_key_name
+    ):
+        grouped_nodes = list(grouped_nodes)
+
+        # The merged node inherits its attributes from the final node in the group.
+        (last_node_key, last_node_value) = grouped_nodes[-1]
+        value_types = set(type(value) for (_, value) in grouped_nodes)
+
+        if len(value_types) > 1:
+            raise ValueError(
+                f'Incompatible types found when trying to merge "{key_name}:" values across configuration files: {", ".join(value_type.id for value_type in value_types)}'
+            )
+
+        # If we're dealing with MappingNodes, recurse and merge its values as well.
+        if ruamel.yaml.nodes.MappingNode in value_types:
+            # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+            # downstream schema validation doesn't break on our application-specific tag.
+            if last_node_value.tag == '!retain' and len(grouped_nodes) > 1:
+                last_node_value.tag = 'tag:yaml.org,2002:map'
+                merged_nodes.append((last_node_key, last_node_value))
+            else:
+                merged_nodes.append(
+                    (
+                        last_node_key,
+                        ruamel.yaml.nodes.MappingNode(
+                            tag=last_node_value.tag,
+                            value=deep_merge_nodes(merge_values(grouped_nodes)),
+                            start_mark=last_node_value.start_mark,
+                            end_mark=last_node_value.end_mark,
+                            flow_style=last_node_value.flow_style,
+                            comment=last_node_value.comment,
+                            anchor=last_node_value.anchor,
+                        ),
                     )
+                )
 
-                # Since we're merging into the B node, consider the A node a duplicate and remove it.
-                replaced_nodes[(a_key, a_value)] = DELETED_NODE
+            continue
 
-                # If we're dealing with MappingNodes, recurse and merge its values as well.
-                if isinstance(b_value, ruamel.yaml.nodes.MappingNode):
-                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
-                    # downstream schema validation doesn't break on our application-specific tag.
-                    if b_value.tag == '!retain':
-                        b_value.tag = 'tag:yaml.org,2002:map'
-                    else:
-                        replaced_nodes[(b_key, b_value)] = (
-                            b_key,
-                            ruamel.yaml.nodes.MappingNode(
-                                tag=b_value.tag,
-                                value=deep_merge_nodes(a_value.value + b_value.value),
-                                start_mark=b_value.start_mark,
-                                end_mark=b_value.end_mark,
-                                flow_style=b_value.flow_style,
-                                comment=b_value.comment,
-                                anchor=b_value.anchor,
-                            ),
-                        )
-                # If we're dealing with SequenceNodes, merge by appending one sequence to the other.
-                elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode):
-                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
-                    # downstream schema validation doesn't break on our application-specific tag.
-                    if b_value.tag == '!retain':
-                        b_value.tag = 'tag:yaml.org,2002:seq'
-                    else:
-                        replaced_nodes[(b_key, b_value)] = (
-                            b_key,
-                            ruamel.yaml.nodes.SequenceNode(
-                                tag=b_value.tag,
-                                value=filter_omitted_nodes(a_value.value + b_value.value),
-                                start_mark=b_value.start_mark,
-                                end_mark=b_value.end_mark,
-                                flow_style=b_value.flow_style,
-                                comment=b_value.comment,
-                                anchor=b_value.anchor,
-                            ),
-                        )
+        # If we're dealing with SequenceNodes, merge by appending sequences together.
+        if ruamel.yaml.nodes.SequenceNode in value_types:
+            if last_node_value.tag == '!retain' and len(grouped_nodes) > 1:
+                last_node_value.tag = 'tag:yaml.org,2002:seq'
+                merged_nodes.append((last_node_key, last_node_value))
+            else:
+                merged_nodes.append(
+                    (
+                        last_node_key,
+                        ruamel.yaml.nodes.SequenceNode(
+                            tag=last_node_value.tag,
+                            value=filter_omitted_nodes(grouped_nodes, merge_values(grouped_nodes)),
+                            start_mark=last_node_value.start_mark,
+                            end_mark=last_node_value.end_mark,
+                            flow_style=last_node_value.flow_style,
+                            comment=last_node_value.comment,
+                            anchor=last_node_value.anchor,
+                        ),
+                    )
+                )
 
-    return [
-        replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE
-    ]
+            continue
+
+        merged_nodes.append((last_node_key, last_node_value))
+
+    return merged_nodes

borgmatic/config/normalize.py

@@ -10,7 +10,11 @@ def normalize_sections(config_filename, config):
 
     Raise ValueError if the "prefix" option is set in both "location" and "consistency" sections.
     '''
-    location = config.get('location') or {}
+    try:
+        location = config.get('location') or {}
+    except AttributeError:
+        raise ValueError('Configuration does not contain any options')
+
     storage = config.get('storage') or {}
     consistency = config.get('consistency') or {}
     hooks = config.get('hooks') or {}

borgmatic/config/schema.yaml

@@ -7,7 +7,7 @@ properties:
         type: object
         description: |
             Constants to use in the configuration file. All occurrences of the
-            constant name within culy braces will be replaced with the value.
+            constant name within curly braces will be replaced with the value.
             For example, if you have a constant named "hostname" with the value
             "myhostname", then the string "{hostname}" will be replaced with
             "myhostname" in the configuration file.
@@ -19,8 +19,8 @@ properties:
         items:
             type: string
         description: |
-            List of source directories and files to backup. Globs and tildes are
-            expanded. Do not backslash spaces in path names.
+            List of source directories and files to back up. Globs and tildes
+            are expanded. Do not backslash spaces in path names.
         example:
             - /home
             - /etc
@@ -29,7 +29,7 @@ properties:
     repositories:
         type: array
         items:
-            type: object 
+            type: object
             required:
                 - path
             properties:
@@ -210,6 +210,13 @@ properties:
             "borgmatic restore" from finding any database dumps created before
             the change. Defaults to ~/.borgmatic
         example: /tmp/borgmatic
+    store_config_files:
+        type: boolean
+        description: |
+            Store configuration files used to create a backup in the backup
+            itself. Defaults to true. Changing this to false prevents "borgmatic
+            bootstrap" from extracting configuration files from the backup.
+        example: true
     source_directories_must_exist:
         type: boolean
         description: |
@@ -253,7 +260,7 @@ properties:
     chunker_params:
         type: string
         description: |
-            Specify the parameters passed to then chunker (CHUNK_MIN_EXP,
+            Specify the parameters passed to the chunker (CHUNK_MIN_EXP,
             CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). See
             https://borgbackup.readthedocs.io/en/stable/internals.html for
             details. Defaults to "19,23,21,4095".
@@ -293,7 +300,7 @@ properties:
         type: string
         description: |
             Command to use instead of "ssh". This can be used to specify ssh
-            options.  Defaults to not set.
+            options. Defaults to not set.
         example: ssh -i /path/to/private/key
     borg_base_directory:
         type: string
@@ -365,19 +372,19 @@ properties:
         type: boolean
         description: |
             Bypass Borg error about a repository that has been moved. Defaults
-            to false.
+            to not bypassing.
         example: true
     unknown_unencrypted_repo_access_is_ok:
         type: boolean
         description: |
             Bypass Borg error about a previously unknown unencrypted repository.
-            Defaults to false.
+            Defaults to not bypassing.
         example: true
     check_i_know_what_i_am_doing:
         type: boolean
         description: |
-            Bypass Borg confirmation about check with repair option.
-            Defaults to false.
+            Bypass Borg confirmation about check with repair option. Defaults to
+            an interactive prompt from Borg.
         example: true
     extra_borg_options:
         type: object
@@ -834,10 +841,121 @@ properties:
         description: |
             List of one or more PostgreSQL databases to dump before creating a
             backup, run once per configuration file. The database dumps are
-            added to your source directories at runtime, backed up, and removed
-            afterwards. Requires pg_dump/pg_dumpall/pg_restore commands. See
+            added to your source directories at runtime and streamed directly
+            to Borg. Requires pg_dump/pg_dumpall/pg_restore commands. See
             https://www.postgresql.org/docs/current/app-pgdump.html and
-            https://www.postgresql.org/docs/current/libpq-ssl.html for details.
+            https://www.postgresql.org/docs/current/libpq-ssl.html for
+            details.
+    mariadb_databases:
+        type: array
+        items:
+            type: object
+            required: ['name']
+            additionalProperties: false
+            properties:
+                name:
+                    type: string
+                    description: |
+                        Database name (required if using this hook). Or "all" to
+                        dump all databases on the host. Note that using this
+                        database hook implicitly enables both read_special and
+                        one_file_system (see above) to support dump and restore
+                        streaming.
+                    example: users
+                hostname:
+                    type: string
+                    description: |
+                        Database hostname to connect to. Defaults to connecting
+                        via local Unix socket.
+                    example: database.example.org
+                restore_hostname:
+                    type: string
+                    description: |
+                        Database hostname to restore to. Defaults to the
+                        "hostname" option.
+                    example: database.example.org
+                port:
+                    type: integer
+                    description: Port to connect to. Defaults to 3306.
+                    example: 3307
+                restore_port:
+                    type: integer
+                    description: |
+                        Port to restore to. Defaults to the "port" option.
+                    example: 5433
+                username:
+                    type: string
+                    description: |
+                        Username with which to connect to the database. Defaults
+                        to the username of the current user.
+                    example: dbuser
+                restore_username:
+                    type: string
+                    description: |
+                        Username with which to restore the database. Defaults to
+                        the "username" option.
+                    example: dbuser
+                password:
+                    type: string
+                    description: |
+                        Password with which to connect to the database. Omitting
+                        a password will only work if MariaDB is configured to
+                        trust the configured username without a password.
+                    example: trustsome1
+                restore_password:
+                    type: string
+                    description: |
+                        Password with which to connect to the restore database.
+                        Defaults to the "password" option.
+                    example: trustsome1
+                format:
+                    type: string
+                    enum: ['sql']
+                    description: |
+                        Database dump output format. Currently only "sql" is
+                        supported. Defaults to "sql" for a single database. Or,
+                        when database name is "all" and format is blank, dumps
+                        all databases to a single file. But if a format is
+                        specified with an "all" database name, dumps each
+                        database to a separate file of that format, allowing
+                        more convenient restores of individual databases.
+                    example: directory
+                add_drop_database:
+                    type: boolean
+                    description: |
+                        Use the "--add-drop-database" flag with mariadb-dump,
+                        causing the database to be dropped right before restore.
+                        Defaults to true.
+                    example: false
+                options:
+                    type: string
+                    description: |
+                        Additional mariadb-dump options to pass directly to the
+                        dump command, without performing any validation on them.
+                        See mariadb-dump documentation for details.
+                    example: --skip-comments
+                list_options:
+                    type: string
+                    description: |
+                        Additional options to pass directly to the mariadb
+                        command that lists available databases, without
+                        performing any validation on them. See mariadb command
+                        documentation for details.
+                    example: --defaults-extra-file=mariadb.cnf
+                restore_options:
+                    type: string
+                    description: |
+                        Additional options to pass directly to the mariadb
+                        command that restores database dumps, without
+                        performing any validation on them. See mariadb command
+                        documentation for details.
+                    example: --defaults-extra-file=mariadb.cnf
+        description: |
+            List of one or more MariaDB databases to dump before creating a
+            backup, run once per configuration file. The database dumps are
+            added to your source directories at runtime and streamed directly
+            to Borg. Requires mariadb-dump/mariadb commands. See
+            https://mariadb.com/kb/en/library/mysqldump/ for details.
     mysql_databases:
         type: array
         items:
@@ -886,7 +1004,7 @@ properties:
                     description: |
                         Username with which to restore the database. Defaults to
                         the "username" option.
-                    example: dbuser                        
+                    example: dbuser
                 password:
                     type: string
                     description: |
@@ -899,7 +1017,7 @@ properties:
                     description: |
                         Password with which to connect to the restore database.
                         Defaults to the "password" option.
-                    example: trustsome1                        
+                    example: trustsome1
                 format:
                     type: string
                     enum: ['sql']
@@ -929,26 +1047,26 @@ properties:
                 list_options:
                     type: string
                     description: |
-                        Additional mysql options to pass directly to the mysql
+                        Additional options to pass directly to the mysql
                         command that lists available databases, without
-                        performing any validation on them. See mysql
+                        performing any validation on them. See mysql command
                         documentation for details.
                     example: --defaults-extra-file=my.cnf
                 restore_options:
                     type: string
                     description: |
-                        Additional mysql options to pass directly to the mysql
-                        command that restores database dumps, without performing
-                        any validation on them. See mysql documentation for
-                        details.
+                        Additional options to pass directly to the mysql
+                        command that restores database dumps, without
+                        performing any validation on them. See mysql command
+                        documentation for details.
                     example: --defaults-extra-file=my.cnf
         description: |
-            List of one or more MySQL/MariaDB databases to dump before creating
-            a backup, run once per configuration file. The database dumps are
-            added to your source directories at runtime, backed up, and removed
-            afterwards. Requires mysqldump/mysql commands (from either MySQL or
-            MariaDB). See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html
-            or https://mariadb.com/kb/en/library/mysqldump/ for details.
+            List of one or more MySQL databases to dump before creating a
+            backup, run once per configuration file. The database dumps are
+            added to your source directories at runtime and streamed directly
+            to Borg. Requires mysqldump/mysql commands. See
+            https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for
+            details.
     sqlite_databases:
         type: array
         items:
@@ -1014,7 +1132,7 @@ properties:
                     type: integer
                     description: |
                         Port to restore to. Defaults to the "port" option.
-                    example: 5433                        
+                    example: 5433
                 username:
                     type: string
                     description: |
@@ -1026,7 +1144,7 @@ properties:
                     description: |
                         Username with which to restore the database. Defaults to
                         the "username" option.
-                    example: dbuser                        
+                    example: dbuser
                 password:
                     type: string
                     description: |
@@ -1073,8 +1191,8 @@ properties:
         description: |
             List of one or more MongoDB databases to dump before creating a
             backup, run once per configuration file. The database dumps are
-            added to your source directories at runtime, backed up, and removed
-            afterwards. Requires mongodump/mongorestore commands. See
+            added to your source directories at runtime and streamed directly
+            to Borg. Requires mongodump/mongorestore commands. See
             https://docs.mongodb.com/database-tools/mongodump/ and
             https://docs.mongodb.com/database-tools/mongorestore/ for details.
     ntfy:
@@ -1268,7 +1386,7 @@ properties:
                 example: a177cad45bd374409f78906a810a3074
         description: |
             Configuration for a monitoring integration with PagerDuty. Create an
-            account at https://www.pagerduty.com/ if you'd like to use this
+            account at https://www.pagerduty.com if you'd like to use this
             service. See borgmatic monitoring documentation for details.
     cronhub:
         type: object

borgmatic/hooks/dispatch.py

@@ -4,6 +4,7 @@ from borgmatic.hooks import (
     cronhub,
     cronitor,
     healthchecks,
+    mariadb,
     mongodb,
     mysql,
     ntfy,
@@ -18,6 +19,7 @@ HOOK_NAME_TO_MODULE = {
     'cronhub': cronhub,
     'cronitor': cronitor,
     'healthchecks': healthchecks,
+    'mariadb_databases': mariadb,
     'mongodb_databases': mongodb,
     'mysql_databases': mysql,
     'ntfy': ntfy,

borgmatic/hooks/dump.py

@@ -7,9 +7,10 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
 logger = logging.getLogger(__name__)
 
 DATABASE_HOOK_NAMES = (
-    'postgresql_databases',
+    'mariadb_databases',
     'mysql_databases',
     'mongodb_databases',
+    'postgresql_databases',
     'sqlite_databases',
 )
 

borgmatic/hooks/mariadb.py

@@ -0,0 +1,230 @@
+import copy
+import logging
+import os
+
+from borgmatic.execute import (
+    execute_command,
+    execute_command_and_capture_output,
+    execute_command_with_processes,
+)
+from borgmatic.hooks import dump
+
+logger = logging.getLogger(__name__)
+
+
+def make_dump_path(config):  # pragma: no cover
+    '''
+    Make the dump path from the given configuration dict and the name of this hook.
+    '''
+    return dump.make_database_dump_path(
+        config.get('borgmatic_source_directory'), 'mariadb_databases'
+    )
+
+
+SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
+
+
+def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
+    '''
+    Given a requested database config, return the corresponding sequence of database names to dump.
+    In the case of "all", query for the names of databases on the configured host and return them,
+    excluding any system databases that will cause problems during restore.
+    '''
+    if database['name'] != 'all':
+        return (database['name'],)
+    if dry_run:
+        return ()
+
+    show_command = (
+        ('mariadb',)
+        + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+        + ('--skip-column-names', '--batch')
+        + ('--execute', 'show schemas')
+    )
+    logger.debug(f'{log_prefix}: Querying for "all" MariaDB databases to dump')
+    show_output = execute_command_and_capture_output(
+        show_command, extra_environment=extra_environment
+    )
+
+    return tuple(
+        show_name
+        for show_name in show_output.strip().splitlines()
+        if show_name not in SYSTEM_DATABASE_NAMES
+    )
+
+
+def execute_dump_command(
+    database, log_prefix, dump_path, database_names, extra_environment, dry_run, dry_run_label
+):
+    '''
+    Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named
+    pipe constructed from the given dump path and database names. Use the given log prefix in any
+    log entries.
+
+    Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if
+    this is a dry run, then don't actually dump anything and return None.
+    '''
+    database_name = database['name']
+    dump_filename = dump.make_database_dump_filename(
+        dump_path, database['name'], database.get('hostname')
+    )
+    if os.path.exists(dump_filename):
+        logger.warning(
+            f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}'
+        )
+        return None
+
+    dump_command = (
+        ('mariadb-dump',)
+        + (tuple(database['options'].split(' ')) if 'options' in database else ())
+        + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+        + ('--databases',)
+        + database_names
+        + ('--result-file', dump_filename)
+    )
+
+    logger.debug(
+        f'{log_prefix}: Dumping MariaDB database "{database_name}" to {dump_filename}{dry_run_label}'
+    )
+    if dry_run:
+        return None
+
+    dump.create_named_pipe_for_dump(dump_filename)
+
+    return execute_command(
+        dump_command,
+        extra_environment=extra_environment,
+        run_to_completion=False,
+    )
+
+
+def dump_databases(databases, config, log_prefix, dry_run):
+    '''
+    Dump the given MariaDB databases to a named pipe. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given
+    configuration dict to construct the destination path and the given log prefix in any log
+    entries.
+
+    Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+    pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
+    '''
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
+
+    logger.info(f'{log_prefix}: Dumping MariaDB databases{dry_run_label}')
+
+    for database in databases:
+        dump_path = make_dump_path(config)
+        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+        dump_database_names = database_names_to_dump(
+            database, extra_environment, log_prefix, dry_run
+        )
+
+        if not dump_database_names:
+            if dry_run:
+                continue
+
+            raise ValueError('Cannot find any MariaDB databases to dump.')
+
+        if database['name'] == 'all' and database.get('format'):
+            for dump_name in dump_database_names:
+                renamed_database = copy.copy(database)
+                renamed_database['name'] = dump_name
+                processes.append(
+                    execute_dump_command(
+                        renamed_database,
+                        log_prefix,
+                        dump_path,
+                        (dump_name,),
+                        extra_environment,
+                        dry_run,
+                        dry_run_label,
+                    )
+                )
+        else:
+            processes.append(
+                execute_dump_command(
+                    database,
+                    log_prefix,
+                    dump_path,
+                    dump_database_names,
+                    extra_environment,
+                    dry_run,
+                    dry_run_label,
+                )
+            )
+
+    return [process for process in processes if process]
+
+
+def remove_database_dumps(databases, config, log_prefix, dry_run):  # pragma: no cover
+    '''
+    Remove all database dump files for this hook regardless of the given databases. Use the given
+    configuration dict to construct the destination path and the log prefix in any log entries. If
+    this is a dry run, then don't actually remove anything.
+    '''
+    dump.remove_database_dumps(make_dump_path(config), 'MariaDB', log_prefix, dry_run)
+
+
+def make_database_dump_pattern(databases, config, log_prefix, name=None):  # pragma: no cover
+    '''
+    Given a sequence of configurations dicts, a configuration dict, a prefix to log with, and a
+    database name to match, return the corresponding glob patterns to match the database dump in an
+    archive.
+    '''
+    return dump.make_database_dump_filename(make_dump_path(config), name, hostname='*')
+
+
+def restore_database_dump(
+    hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
+):
+    '''
+    Restore a database from the given extract stream. The database is supplied as a configuration
+    dict, but the given hook configuration is ignored. The given configuration dict is used to
+    construct the destination path, and the given log prefix is used for any log entries. If this is
+    a dry run, then don't actually restore anything. Trigger the given active extract process (an
+    instance of subprocess.Popen) to produce output to consume.
+    '''
+    dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+    hostname = connection_params['hostname'] or database.get(
+        'restore_hostname', database.get('hostname')
+    )
+    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
+    username = connection_params['username'] or database.get(
+        'restore_username', database.get('username')
+    )
+    password = connection_params['password'] or database.get(
+        'restore_password', database.get('password')
+    )
+
+    restore_command = (
+        ('mariadb', '--batch')
+        + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+        + (('--host', hostname) if hostname else ())
+        + (('--port', str(port)) if port else ())
+        + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--user', username) if username else ())
+    )
+    extra_environment = {'MYSQL_PWD': password} if password else None
+
+    logger.debug(f"{log_prefix}: Restoring MariaDB database {database['name']}{dry_run_label}")
+    if dry_run:
+        return
+
+    # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
+    # if the restore paths don't exist in the archive.
+    execute_command_with_processes(
+        restore_command,
+        [extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=extra_environment,
+    )

borgmatic/hooks/mongodb.py

@@ -59,26 +59,23 @@ def build_dump_command(database, dump_filename, dump_format):
     Return the mongodump command from a single database configuration.
     '''
     all_databases = database['name'] == 'all'
-    command = ['mongodump']
-    if dump_format == 'directory':
-        command.extend(('--out', dump_filename))
-    if 'hostname' in database:
-        command.extend(('--host', database['hostname']))
-    if 'port' in database:
-        command.extend(('--port', str(database['port'])))
-    if 'username' in database:
-        command.extend(('--username', database['username']))
-    if 'password' in database:
-        command.extend(('--password', database['password']))
-    if 'authentication_database' in database:
-        command.extend(('--authenticationDatabase', database['authentication_database']))
-    if not all_databases:
-        command.extend(('--db', database['name']))
-    if 'options' in database:
-        command.extend(database['options'].split(' '))
-    if dump_format != 'directory':
-        command.extend(('--archive', '>', dump_filename))
-    return command
+
+    return (
+        ('mongodump',)
+        + (('--out', dump_filename) if dump_format == 'directory' else ())
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--username', database['username']) if 'username' in database else ())
+        + (('--password', database['password']) if 'password' in database else ())
+        + (
+            ('--authenticationDatabase', database['authentication_database'])
+            if 'authentication_database' in database
+            else ()
+        )
+        + (('--db', database['name']) if not all_databases else ())
+        + (tuple(database['options'].split(' ')) if 'options' in database else ())
+        + (('--archive', '>', dump_filename) if dump_format != 'directory' else ())
+    )
 
 
 def remove_database_dumps(databases, config, log_prefix, dry_run):  # pragma: no cover
@@ -100,32 +97,19 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None):  # pra
 
 
 def restore_database_dump(
-    databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params
+    hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
 ):
     '''
-    Restore the given MongoDB database from an extract stream. The databases are supplied as a
-    sequence containing one dict describing each database (as per the configuration schema), but
-    only the database corresponding to the given database name is restored. Use the configuration
-    dict to construct the destination path and the given log prefix in any log entries. If this is a
-    dry run, then don't actually restore anything. Trigger the given active extract process (an
+    Restore a database from the given extract stream. The database is supplied as a configuration
+    dict, but the given hook configuration is ignored. The given configuration dict is used to
+    construct the destination path, and the given log prefix is used for any log entries. If this is
+    a dry run, then don't actually restore anything. Trigger the given active extract process (an
     instance of subprocess.Popen) to produce output to consume.
 
     If the extract process is None, then restore the dump from the filesystem rather than from an
     extract stream.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-
-    try:
-        database = next(
-            database_config
-            for database_config in databases_config
-            if database_config.get('name') == database_name
-        )
-    except StopIteration:
-        raise ValueError(
-            f'A database named "{database_name}" could not be found in the configuration'
-        )
-
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(config), database['name'], database.get('hostname')
     )
@@ -168,7 +152,7 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
     else:
         command.extend(('--dir', dump_filename))
     if database['name'] != 'all':
-        command.extend(('--drop', '--db', database['name']))
+        command.extend(('--drop',))
     if hostname:
         command.extend(('--host', hostname))
     if port:
@@ -181,7 +165,8 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
         command.extend(('--authenticationDatabase', database['authentication_database']))
     if 'restore_options' in database:
         command.extend(database['restore_options'].split(' '))
-    if database['schemas']:
+    if database.get('schemas'):
         for schema in database['schemas']:
             command.extend(('--nsInclude', schema))
+
     return command

borgmatic/hooks/mysql.py

@@ -181,28 +181,16 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None):  # pra
 
 
 def restore_database_dump(
-    databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params
+    hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
 ):
     '''
-    Restore the given MySQL/MariaDB database from an extract stream. The databases are supplied as a
-    sequence containing one dict describing each database (as per the configuration schema), but
-    only the database corresponding to the given database name is restored. Use the given log
-    prefix in any log entries. If this is a dry run, then don't actually restore anything. Trigger
-    the given active extract process (an instance of subprocess.Popen) to produce output to consume.
+    Restore a database from the given extract stream. The database is supplied as a configuration
+    dict, but the given hook configuration is ignored. The given configuration dict is used to
+    construct the destination path, and the given log prefix is used for any log entries. If this is
+    a dry run, then don't actually restore anything. Trigger the given active extract process (an
+    instance of subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-
-    try:
-        database = next(
-            database_config
-            for database_config in databases_config
-            if database_config.get('name') == database_name
-        )
-    except StopIteration:
-        raise ValueError(
-            f'A database named "{database_name}" could not be found in the configuration'
-        )
-
     hostname = connection_params['hostname'] or database.get(
         'restore_hostname', database.get('hostname')
     )

borgmatic/hooks/ntfy.py

@@ -28,8 +28,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
         state_config = hook_config.get(
             state.name.lower(),
             {
-                'title': f'A Borgmatic {state.name} event happened',
-                'message': f'A Borgmatic {state.name} event happened',
+                'title': f'A borgmatic {state.name} event happened',
+                'message': f'A borgmatic {state.name} event happened',
                 'priority': 'default',
                 'tags': 'borgmatic',
             },

borgmatic/hooks/postgresql.py

@@ -202,15 +202,14 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None):  # pra
 
 
 def restore_database_dump(
-    databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params
+    hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
 ):
     '''
-    Restore the given PostgreSQL database from an extract stream. The databases are supplied as a
-    sequence containing one dict describing each database (as per the configuration schema), but
-    only the database corresponding to the given database name is restored. Use the given
-    configuration dict to construct the destination path and the given log prefix in any log
-    entries. If this is a dry run, then don't actually restore anything. Trigger the given active
-    extract process (an instance of subprocess.Popen) to produce output to consume.
+    Restore a database from the given extract stream. The database is supplied as a configuration
+    dict, but the given hook configuration is ignored. The given configuration dict is used to
+    construct the destination path, and the given log prefix is used for any log entries. If this is
+    a dry run, then don't actually restore anything. Trigger the given active extract process (an
+    instance of subprocess.Popen) to produce output to consume.
 
     If the extract process is None, then restore the dump from the filesystem rather than from an
     extract stream.
@@ -219,18 +218,6 @@ def restore_database_dump(
     hostname, port, username, and password.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-
-    try:
-        database = next(
-            database_config
-            for database_config in databases_config
-            if database_config.get('name') == database_name
-        )
-    except StopIteration:
-        raise ValueError(
-            f'A database named "{database_name}" could not be found in the configuration'
-        )
-
     hostname = connection_params['hostname'] or database.get(
         'restore_hostname', database.get('hostname')
     )

borgmatic/hooks/sqlite.py

@@ -32,10 +32,10 @@ def dump_databases(databases, config, log_prefix, dry_run):
         database_path = database['path']
 
         if database['name'] == 'all':
-            logger.warning('The "all" database name has no meaning for SQLite3 databases')
+            logger.warning('The "all" database name has no meaning for SQLite databases')
         if not os.path.exists(database_path):
             logger.warning(
-                f'{log_prefix}: No SQLite database at {database_path}; An empty database will be created and dumped'
+                f'{log_prefix}: No SQLite database at {database_path}; an empty database will be created and dumped'
             )
 
         dump_path = make_dump_path(config)
@@ -84,28 +84,16 @@ def make_database_dump_pattern(databases, config, log_prefix, name=None):  # pra
 
 
 def restore_database_dump(
-    databases_config, config, log_prefix, database_name, dry_run, extract_process, connection_params
+    hook_config, config, log_prefix, database, dry_run, extract_process, connection_params
 ):
     '''
-    Restore the given SQLite3 database from an extract stream. The databases are supplied as a
-    sequence containing one dict describing each database (as per the configuration schema), but
-    only the database corresponding to the given database name is restored. Use the given log prefix
-    in any log entries. If this is a dry run, then don't actually restore anything. Trigger the
-    given active extract process (an instance of subprocess.Popen) to produce output to consume.
+    Restore a database from the given extract stream. The database is supplied as a configuration
+    dict, but the given hook configuration is ignored. The given configuration dict is used to
+    construct the destination path, and the given log prefix is used for any log entries. If this is
+    a dry run, then don't actually restore anything. Trigger the given active extract process (an
+    instance of subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-
-    try:
-        database = next(
-            database_config
-            for database_config in databases_config
-            if database_config.get('name') == database_name
-        )
-    except StopIteration:
-        raise ValueError(
-            f'A database named "{database_name}" could not be found in the configuration'
-        )
-
     database_path = connection_params['restore_path'] or database.get(
         'restore_path', database.get('path')
     )

docs/SECURITY.md

@@ -16,4 +16,4 @@ each.
 If you find a security vulnerability, please [file a
 ticket](https://torsion.org/borgmatic/#issues) or [send email
 directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
-back within a few days at most, and generally sooner.
+back within a few days at most and generally sooner.

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -17,7 +17,7 @@ feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
 instead.
 
 You can specify `before_backup` hooks to perform preparation steps before
-running backups, and specify `after_backup` hooks to perform cleanup steps
+running backups and specify `after_backup` hooks to perform cleanup steps
 afterwards. Here's an example:
 
 ```yaml

docs/how-to/backup-your-databases.md

@@ -15,7 +15,7 @@ consistent snapshot that is more suited for backups.
 
 Fortunately, borgmatic includes built-in support for creating database dumps
 prior to running backups. For example, here is everything you need to dump and
-backup a couple of local PostgreSQL databases and a MySQL/MariaDB database.
+backup a couple of local PostgreSQL databases and a MySQL database.
 
 ```yaml
 postgresql_databases:
@@ -46,6 +46,16 @@ sqlite_databases:
       path: /var/lib/sqlite3/mydb.sqlite
 ```
 
+<span class="minilink minilink-addedin">New in version 1.8.2</span> If you're
+using MariaDB, use the MariaDB database hook instead of `mysql_databases:` as
+the MariaDB hook calls native MariaDB commands instead of the deprecated MySQL
+ones. For instance:
+
+```yaml
+mariadb_databases:
+    - name: comments
+```
+
 As part of each backup, borgmatic streams a database dump for each configured
 database directly to Borg, so it's included in the backup without consuming
 additional disk space. (The exceptions are the PostgreSQL/MongoDB "directory"
@@ -75,16 +85,23 @@ postgresql_databases:
       password: trustsome1
       format: tar
       options: "--role=someone"
+mariadb_databases:
+    - name: photos
+      hostname: database3.example.org
+      port: 3307
+      username: root
+      password: trustsome1
+      options: "--skip-comments"
 mysql_databases:
     - name: posts
-      hostname: database3.example.org
+      hostname: database4.example.org
       port: 3307
       username: root
       password: trustsome1
       options: "--skip-comments"
 mongodb_databases:
     - name: messages
-      hostname: database4.example.org
+      hostname: database5.example.org
       port: 27018
       username: dbuser
       password: trustsome1
@@ -108,6 +125,8 @@ If you want to dump all databases on a host, use `all` for the database name:
 ```yaml
 postgresql_databases:
     - name: all
+mariadb_databases:
+    - name: all
 mysql_databases:
     - name: all
 mongodb_databases:
@@ -123,15 +142,18 @@ The SQLite hook in particular does not consider "all" a special database name.
 these options in the `hooks:` section of your configuration.
 
 <span class="minilink minilink-addedin">New in version 1.7.6</span> With
-PostgreSQL and MySQL, you can optionally dump "all" databases to separate
-files instead of one combined dump file, allowing more convenient restores of
-individual databases. Enable this by specifying your desired database dump
-`format`:
+PostgreSQL, MariaDB, and MySQL, you can optionally dump "all" databases to
+separate files instead of one combined dump file, allowing more convenient
+restores of individual databases. Enable this by specifying your desired
+database dump `format`:
 
 ```yaml
 postgresql_databases:
     - name: all
       format: custom
+mariadb_databases:
+    - name: all
+      format: sql
 mysql_databases:
     - name: all
       format: sql
@@ -222,10 +244,16 @@ to prepare for this situation, it's a good idea to include borgmatic's own
 configuration files as part of your regular backups. That way, you can always
 bring back any missing configuration files in order to restore a database.
 
+<span class="minilink minilink-addedin">New in version 1.7.15</span> borgmatic
+automatically includes configuration files in your backup. See [the
+documentation on the `config bootstrap`
+action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive)
+for more information.
+
 
 ## Supported databases
 
-As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB, and SQLite
+As of now, borgmatic supports PostgreSQL, MariaDB, MySQL, MongoDB, and SQLite
 databases directly. But see below about general-purpose preparation and
 cleanup hooks as a work-around with other database systems. Also, please [file
 a ticket](https://torsion.org/borgmatic/#issues) for additional database
@@ -282,7 +310,8 @@ problem: the `restore` action figures out which repository to use.
 
 But if you have multiple repositories configured, then you'll need to specify
 the repository to use via the `--repository` flag. This can be done either
-with the repository's path or its label as configured in your borgmatic configuration file.
+with the repository's path or its label as configured in your borgmatic
+configuration file.
 
 ```bash
 borgmatic restore --repository repo.borg --archive host-2023-...
@@ -338,6 +367,28 @@ within the database dump:
 borgmatic restore --archive latest --database users --schema tentant1
 ```
 
+### Restore to an alternate host
+<span class="minilink minilink-addedin">New in version 1.7.15</span>
+A database dump can be restored to a host other than the one from which it was
+originally dumped. The connection parameters like the username, password, and
+port can also be changed. This can be done from the command line:
+
+```bash
+borgmatic restore --archive latest --database users --hostname database2.example.org --port 5433 --username postgres --password trustsome1
+```
+
+Or from the configuration file:
+
+```yaml
+postgresql_databases:
+    - name: users
+        hostname: database1.example.org
+        restore_hostname: database1.example.org
+        restore_port: 5433
+        restore_username: postgres
+        restore_password: trustsome1
+```
+
 
 ### Limitations
 
@@ -398,9 +449,9 @@ dumps with any database system.
 
 ## Troubleshooting
 
-### PostgreSQL/MySQL authentication errors
+### Authentication errors
 
-With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors
+With PostgreSQL, MariaDB, and MySQL, if you're getting authentication errors
 when borgmatic tries to connect to your database, a natural reaction is to
 increase your borgmatic verbosity with `--verbosity 2` and go looking in the
 logs. You'll notice though that your database password does not show up in the
@@ -414,23 +465,24 @@ authenticated. For instance, with PostgreSQL, check your
 [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html)
 file for that configuration.
 
-Additionally, MySQL/MariaDB may be picking up some of your credentials from a
-defaults file like `~/.my.cnf`. If that's the case, then it's possible
-MySQL/MariaDB ends up using, say, a username from borgmatic's configuration
-and a password from `~/.my.cnf`. This may result in authentication errors if
-this combination of credentials is not what you intend.
+Additionally, MariaDB or MySQL may be picking up some of your credentials from
+a defaults file like `~/mariadb.cnf` or `~/.my.cnf`. If that's the case, then
+it's possible MariaDB or MySQL end up using, say, a username from borgmatic's
+configuration and a password from `~/mariadb.cnf` or `~/.my.cnf`. This may
+result in authentication errors if this combination of credentials is not what
+you intend.
 
 
-### MySQL table lock errors
+### MariaDB or MySQL table lock errors
 
-If you encounter table lock errors during a database dump with MySQL/MariaDB,
-you may need to [use a
-transaction](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html#option_mysqldump_single-transaction).
+If you encounter table lock errors during a database dump with MariaDB or
+MySQL, you may need to [use a
+transaction](https://mariadb.com/docs/skysql-dbaas/ref/mdb/cli/mariadb-dump/single-transaction/).
 You can add any additional flags to the `options:` in your database
-configuration. Here's an example:
+configuration. Here's an example with MariaDB:
 
 ```yaml
-mysql_databases:
+mariadb_databases:
     - name: posts
       options: "--single-transaction --quick"
 ```

docs/how-to/deal-with-very-large-backups.md

@@ -189,7 +189,7 @@ check_repositories:
     - path/of/repository_to_check.borg
 ```
 
-Finally, you can override your configuration file's consistency checks, and
+Finally, you can override your configuration file's consistency checks and
 run particular checks via the command-line. For instance:
 
 ```bash

docs/how-to/develop-on-borgmatic.md

@@ -37,6 +37,9 @@ install (so without `--user`, as root), or even into a
 borgmatic is up to you, but generally an editable install makes development
 and testing easier.
 
+To get oriented with the borgmatic source code, have a look at the [source
+code reference](https://torsion.org/borgmatic/docs/reference/source-code/).
+
 
 ## Automated tests
 
@@ -149,8 +152,8 @@ See the Black, Flake8, and isort documentation for more information.
 
 Each pull request triggers a continuous integration build which runs the test
 suite. You can view these builds on
-[build.torsion.org](https://build.torsion.org/borgmatic-collective/borgmatic), and they're
-also linked from the commits list on each pull request.
+[build.torsion.org](https://build.torsion.org/borgmatic-collective/borgmatic),
+and they're also linked from the commits list on each pull request.
 
 ## Documentation development
 

docs/how-to/extract-a-backup.md

@@ -44,7 +44,6 @@ entire contents of the archive to the current directory, so make sure you're
 in the right place before running the command—or see below about the
 `--destination` flag.
 
-
 ## Repository selection
 
 If you have a single repository in your borgmatic configuration file(s), no
@@ -145,3 +144,46 @@ When you're all done exploring your files, unmount your mount point. No
 ```bash
 borgmatic umount --mount-point /mnt
 ```
+
+## Extract the configuration files used to create an archive
+
+<span class="minilink minilink-addedin">New in version 1.7.15</span> borgmatic
+automatically stores all the configuration files used to create an archive
+inside the archive itself. This is useful in cases where you've lost a
+configuration file or you want to see what configurations were used to create a
+particular archive.
+
+To extract the configuration files from an archive, use the `config bootstrap`
+action. For example:
+
+```bash 
+borgmatic config bootstrap --repository repo.borg --destination /tmp
+```
+
+This extracts the configuration file from the latest archive in the repository
+`repo.borg` to `/tmp/etc/borgmatic/config.yaml`, assuming that the only
+configuration file used to create this archive was located at
+`/etc/borgmatic/config.yaml` when the archive was created.
+
+Note that to run the `config bootstrap` action, you don't need to have a
+borgmatic configuration file. You only need to specify the repository to use via
+the `--repository` flag; borgmatic will figure out the rest.
+
+If a destination directory is not specified, the configuration files will be
+extracted to their original locations, silently *overwriting* any configuration
+files that may already exist. For example, if a configuration file was located
+at `/etc/borgmatic/config.yaml` when the archive was created, it will be
+extracted to `/etc/borgmatic/config.yaml` too.
+
+If you want to extract the configuration file from a specific archive, use the
+`--archive` flag:
+
+```bash
+borgmatic config bootstrap --repository repo.borg --archive host-2023-01-02T04:06:07.080910 --destination /tmp
+```
+
+<span class="minilink minilink-addedin">New in version 1.8.1</span> Set the
+`store_config_files` option to `false` to disable the automatic backup of
+borgmatic configuration files, for instance if they contain sensitive
+information you don't want to store even inside your encrypted backups. If you
+do this though, the `config bootstrap` action will no longer work.

docs/how-to/make-per-application-backups.md

@@ -153,7 +153,7 @@ in newer versions of borgmatic.
 Once you have multiple different configuration files, you might want to share
 common configuration options across these files with having to copy and paste
 them. To achieve this, you can put fragments of common configuration options
-into a file, and then include or inline that file into one or more borgmatic
+into a file and then include or inline that file into one or more borgmatic
 configuration files.
 
 Let's say that you want to include common consistency check configuration across all
@@ -233,18 +233,57 @@ checks:
 <span class="minilink minilink-addedin">Prior to version 1.8.0</span> These
 options were organized into sections like `retention:` and `consistency:`.
 
-Once this include gets merged in, the resulting configuration would have all
-of the options from the original configuration file *and* the options from the
+Once this include gets merged in, the resulting configuration has all of the
+options from the original configuration file *and* the options from the
 include.
 
 Note that this `<<` include merging syntax is only for merging in mappings
-(configuration options and their values). But if you'd like to include a
-single value directly, please see the above about standard includes.
+(configuration options and their values). If you'd like to include a single
+value directly, please see above about standard includes.
 
-Additionally, there is a limitation preventing multiple `<<` include merges
-per file or option value. So for instance, that means you can do one `<<`
-merge at the global level, another `<<` within each nested option value, etc.
-(This is a YAML limitation.)
+
+### Multiple merge includes
+
+borgmatic has a limitation preventing multiple `<<` include merges per file or
+option value. This means you can do a single `<<` merge at the global level,
+another `<<` within each nested option value, etc. (This is a YAML
+limitation.) For instance:
+
+```yaml
+repositories:
+   - path: repo.borg
+
+# This won't work! You can't do multiple merges like this at the same level.
+<<: !include common1.yaml
+<<: !include common2.yaml
+```
+
+But read on for a way around this.
+
+<span class="minilink minilink-addedin">New in version 1.8.1</span> You can
+include and merge multiple configuration files all at once. For instance:
+
+```yaml
+repositories:
+   - path: repo.borg
+
+<<: !include [common1.yaml, common2.yaml, common3.yaml]
+```
+
+This merges in each included configuration file in turn, such that later files
+replace the options in earlier ones.
+
+Here's another way to do the same thing:
+
+```yaml
+repositories:
+   - path: repo.borg
+
+<<: !include
+    - common1.yaml
+    - common2.yaml
+    - common3.yaml
+```
 
 
 ### Deep merge
@@ -309,8 +348,8 @@ source_directories:
     - /etc
 ```
 
-<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
-this option in the `location:` section of your configuration.
+<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put the
+`source_directories` option in the `location:` section of your configuration.
 
 Once this include gets merged in, the resulting configuration will have a
 `source_directories` value of `/etc` and `/var`—with `/home` omitted.
@@ -442,8 +481,8 @@ command-line via the `--override` flag. Here's an example:
 borgmatic create --override remote_path=/usr/local/bin/borg1
 ```
 
-What this does is load your configuration files, and for each one, disregard
-the configured value for the `remote_path` option, and use the value of
+What this does is load your configuration files and for each one, disregard
+the configured value for the `remote_path` option and use the value of
 `/usr/local/bin/borg1` instead.
 
 You can even override nested values or multiple values at once. For instance:

docs/how-to/monitor-your-backups.md

@@ -106,7 +106,7 @@ on_error:
 ```
 
 In this example, when the error occurs, borgmatic interpolates runtime values
-into the hook command: the borgmatic configuration filename, and the path of
+into the hook command: the borgmatic configuration filename and the path of
 the repository. Here's the full set of supported variables you can use here:
 
  * `configuration_filename`: borgmatic configuration filename in which the
@@ -118,7 +118,7 @@ the repository. Here's the full set of supported variables you can use here:
    occurred without running a command)
 
 Note that borgmatic runs the `on_error` hooks only for `create`, `prune`,
-`compact`, or `check` actions or hooks in which an error occurs, and not other
+`compact`, or `check` actions/hooks in which an error occurs and not other
 actions. borgmatic does not run `on_error` hooks if an error occurs within a
 `before_everything` or `after_everything` hook. For more about hooks, see the
 [borgmatic hooks
@@ -150,7 +150,7 @@ hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
 the `create`, `prune`, `compact`, or `check` actions are run.
 
 Then, if the actions complete successfully, borgmatic notifies Healthchecks of
-the success after the `after_backup` hooks run, and includes borgmatic logs in
+the success after the `after_backup` hooks run and includes borgmatic logs in
 the payload data sent to Healthchecks. This means that borgmatic logs show up
 in the Healthchecks UI, although be aware that Healthchecks currently has a
 10-kilobyte limit for the logs in each ping.
@@ -304,17 +304,17 @@ ntfy:
     topic: my-unique-topic
     server: https://ntfy.my-domain.com
     start:
-        title: A Borgmatic backup started
+        title: A borgmatic backup started
         message: Watch this space...
         tags: borgmatic
         priority: min
     finish:
-        title: A Borgmatic backup completed successfully
+        title: A borgmatic backup completed successfully
         message: Nice!
         tags: borgmatic,+1
         priority: min
     fail:
-        title: A Borgmatic backup failed
+        title: A borgmatic backup failed
         message: You should probably fix it
         tags: borgmatic,-1,skull
         priority: max
@@ -336,7 +336,7 @@ output formatted as JSON.
 
 Note that when you specify the `--json` flag, Borg's other non-JSON output is
 suppressed so as not to interfere with the captured JSON. Also note that JSON
-output only shows up at the console, and not in syslog.
+output only shows up at the console and not in syslog.
 
 
 ### Latest backups

docs/how-to/set-up-backups.md

@@ -59,11 +59,11 @@ borgmatic's path.
 
 ### Global install option
 
-If you try the user site installation above, and have problems making
-borgmatic commands runnable on your system `PATH`, an alternate approach is to
-install borgmatic globally.
+If you try the user site installation above and have problems making borgmatic
+commands runnable on your system `PATH`, an alternate approach is to install
+borgmatic globally.
 
-The following uninstalls borgmatic, and then reinstalls it such that borgmatic
+The following uninstalls borgmatic and then reinstalls it such that borgmatic
 commands are on the default system `PATH`:
 
 ```bash
@@ -229,7 +229,7 @@ sudo borgmatic rcreate --encryption repokey-aes-ocb
 certain platforms like ARM64.)
 
 This uses the borgmatic configuration file you created above to determine
-which local or remote repository to create, and encrypts it with the
+which local or remote repository to create and encrypts it with the
 encryption passphrase specified there if one is provided. Read about [Borg
 encryption
 modes](https://borgbackup.readthedocs.io/en/stable/usage/init.html#encryption-mode-tldr)

docs/how-to/upgrade.md

@@ -63,7 +63,7 @@ and, if desired, replace your original configuration file with it.
 ### Upgrading from borgmatic 1.0.x
 
 borgmatic changed its configuration file format in version 1.1.0 from
-INI-style to YAML. This better supports validation, and has a more natural way
+INI-style to YAML. This better supports validation and has a more natural way
 to express lists of values. To upgrade your existing configuration, first
 upgrade to the last version of borgmatic to support converting configuration:
 borgmatic 1.7.14.

docs/reference/source-code.md

@@ -0,0 +1,29 @@
+---
+title: Source code reference
+eleventyNavigation:
+  key: 🐍 Source code reference
+  parent: Reference guides
+  order: 3
+---
+## getting oriented
+
+If case you're interested in [developing on
+borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/),
+here's an abridged primer on how its Python source code is organized to help
+you get started. At the top level we have:
+
+ * [borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic): The main borgmatic source module. Most of the code is here.
+ * [docs](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/docs): How-to and reference documentation, including the document you're reading now.
+ * [sample](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/sample): Example configurations for cron and systemd.
+ * [scripts](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/scripts): Dev-facing scripts for things like building documentation and running end-to-end tests.
+ * [tests](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/tests): Automated tests organized by: end-to-end, integration, and unit.
+
+Within the `borgmatic` directory you'll find:
+
+ * [actions](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/actions): Mid-level code for running each borgmatic action (create, list, check, etc.).
+ * [borg](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/borg): Lower-level code that actually shells out to Borg for each action.
+ * [commands](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/commands): Looking to add a new flag or action? Start here. This contains borgmatic's entry point, argument parsing, and shell completion. 
+ * [config](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/config): Code responsible for loading, normalizing, and validating borgmatic's configuration.
+ * [hooks](https://projects.torsion.org/borgmatic-collective/borgmatic/src/branch/main/borgmatic/hooks): Looking to add a new database or monitoring integration? Start here.
+
+So, broadly speaking, the control flow goes: `commands` → `config` followed by `commands` → `actions` → `borg` and `hooks`.

scripts/run-full-tests

@@ -21,7 +21,7 @@ apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client m
     py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite fish
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
-python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1
+python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 pymongo==4.4.1
 pip3 install --ignore-installed tox==3.25.1
 export COVERAGE_FILE=/tmp/.coverage
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.8.0'
+VERSION = '1.8.2'
 
 
 setup(

tests/end-to-end/docker-compose.yaml

@@ -10,18 +10,28 @@ services:
     environment:
       POSTGRES_PASSWORD: test2
       POSTGRES_DB: test
-      POSTGRES_USER: postgres2
     command: docker-entrypoint.sh -p 5433
-  mysql:
-    image: docker.io/mariadb:10.5
+  mariadb:
+    image: docker.io/mariadb:10.11.4
     environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-  mysql2:
-    image: docker.io/mariadb:10.5
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: test
+  mariadb2:
+    image: docker.io/mariadb:10.11.4
     environment:
-      MYSQL_ROOT_PASSWORD: test2
-      MYSQL_DATABASE: test
+      MARIADB_ROOT_PASSWORD: test2
+      MARIADB_DATABASE: test
+    command: docker-entrypoint.sh --port=3307
+  not-actually-mysql:
+    image: docker.io/mariadb:10.11.4
+    environment:
+      MARIADB_ROOT_PASSWORD: test
+      MARIADB_DATABASE: test
+  not-actually-mysql2:
+    image: docker.io/mariadb:10.11.4
+    environment:
+      MARIADB_ROOT_PASSWORD: test2
+      MARIADB_DATABASE: test
     command: docker-entrypoint.sh --port=3307
   mongodb:
     image: docker.io/mongo:5.0.5
@@ -50,7 +60,7 @@ services:
     depends_on:
       - postgresql
       - postgresql2
-      - mysql
-      - mysql2
+      - mariadb
+      - mariadb2
       - mongodb
       - mongodb2

tests/end-to-end/test_database.py

@@ -5,7 +5,9 @@ import subprocess
 import sys
 import tempfile
 
+import pymongo
 import pytest
+import ruamel.yaml
 
 
 def write_configuration(
@@ -21,7 +23,7 @@ def write_configuration(
     for testing. This includes injecting the given repository path, borgmatic source directory for
     storing database dumps, dump format (for PostgreSQL), and encryption passphrase.
     '''
-    config = f'''
+    config_yaml = f'''
 source_directories:
     - {source_directory}
 repositories:
@@ -45,18 +47,32 @@ postgresql_databases:
       hostname: postgresql
       username: postgres
       password: test
-mysql_databases:
+mariadb_databases:
     - name: test
-      hostname: mysql
+      hostname: mariadb
       username: root
       password: test
     - name: all
-      hostname: mysql
+      hostname: mariadb
       username: root
       password: test
     - name: all
       format: sql
-      hostname: mysql
+      hostname: mariadb
+      username: root
+      password: test
+mysql_databases:
+    - name: test
+      hostname: not-actually-mysql
+      username: root
+      password: test
+    - name: all
+      hostname: not-actually-mysql
+      username: root
+      password: test
+    - name: all
+      format: sql
+      hostname: not-actually-mysql
       username: root
       password: test
 mongodb_databases:
@@ -76,7 +92,9 @@ sqlite_databases:
 '''
 
     with open(config_path, 'w') as config_file:
-        config_file.write(config)
+        config_file.write(config_yaml)
+
+    return ruamel.yaml.YAML(typ='safe').load(config_yaml)
 
 
 def write_custom_restore_configuration(
@@ -92,7 +110,7 @@ def write_custom_restore_configuration(
     for testing with custom restore options. This includes a custom restore_hostname, restore_port,
     restore_username, restore_password and restore_path.
     '''
-    config = f'''
+    config_yaml = f'''
 source_directories:
     - {source_directory}
 repositories:
@@ -109,14 +127,22 @@ postgresql_databases:
       format: {postgresql_dump_format}
       restore_hostname: postgresql2
       restore_port: 5433
-      restore_username: postgres2
+      restore_password: test2
+mariadb_databases:
+    - name: test
+      hostname: mariadb
+      username: root
+      password: test
+      restore_hostname: mariadb2
+      restore_port: 3307
+      restore_username: root
       restore_password: test2
 mysql_databases:
     - name: test
-      hostname: mysql
+      hostname: not-actually-mysql
       username: root
       password: test
-      restore_hostname: mysql2
+      restore_hostname: not-actually-mysql2
       restore_port: 3307
       restore_username: root
       restore_password: test2
@@ -138,7 +164,9 @@ sqlite_databases:
 '''
 
     with open(config_path, 'w') as config_file:
-        config_file.write(config)
+        config_file.write(config_yaml)
+
+    return ruamel.yaml.YAML(typ='safe').load(config_yaml)
 
 
 def write_simple_custom_restore_configuration(
@@ -154,7 +182,7 @@ def write_simple_custom_restore_configuration(
     custom restore_hostname, restore_port, restore_username and restore_password as we only test
     these options for PostgreSQL.
     '''
-    config = f'''
+    config_yaml = f'''
 source_directories:
     - {source_directory}
 repositories:
@@ -172,7 +200,147 @@ postgresql_databases:
 '''
 
     with open(config_path, 'w') as config_file:
-        config_file.write(config)
+        config_file.write(config_yaml)
+
+    return ruamel.yaml.YAML(typ='safe').load(config_yaml)
+
+
+def get_connection_params(database, use_restore_options=False):
+    hostname = (database.get('restore_hostname') if use_restore_options else None) or database.get(
+        'hostname'
+    )
+    port = (database.get('restore_port') if use_restore_options else None) or database.get('port')
+    username = (database.get('restore_username') if use_restore_options else None) or database.get(
+        'username'
+    )
+    password = (database.get('restore_password') if use_restore_options else None) or database.get(
+        'password'
+    )
+
+    return (hostname, port, username, password)
+
+
+def run_postgresql_command(command, config, use_restore_options=False):
+    (hostname, port, username, password) = get_connection_params(
+        config['postgresql_databases'][0], use_restore_options
+    )
+
+    subprocess.check_call(
+        [
+            '/usr/bin/psql',
+            f'--host={hostname}',
+            f'--port={port or 5432}',
+            f"--username={username or 'root'}",
+            f'--command={command}',
+            'test',
+        ],
+        env={'PGPASSWORD': password},
+    )
+
+
+def run_mariadb_command(command, config, use_restore_options=False, binary_name='mariadb'):
+    (hostname, port, username, password) = get_connection_params(
+        config[f'{binary_name}_databases'][0], use_restore_options
+    )
+
+    subprocess.check_call(
+        [
+            f'/usr/bin/{binary_name}',
+            f'--host={hostname}',
+            f'--port={port or 3306}',
+            f'--user={username}',
+            f'--execute={command}',
+            'test',
+        ],
+        env={'MYSQL_PWD': password},
+    )
+
+
+def get_mongodb_database_client(config, use_restore_options=False):
+    (hostname, port, username, password) = get_connection_params(
+        config['mongodb_databases'][0], use_restore_options
+    )
+
+    return pymongo.MongoClient(f'mongodb://{username}:{password}@{hostname}:{port or 27017}').test
+
+
+def run_sqlite_command(command, config, use_restore_options=False):
+    database = config['sqlite_databases'][0]
+    path = (database.get('restore_path') if use_restore_options else None) or database.get('path')
+
+    subprocess.check_call(
+        [
+            '/usr/bin/sqlite3',
+            path,
+            command,
+            '.exit',
+        ],
+    )
+
+
+DEFAULT_HOOK_NAMES = {'postgresql', 'mariadb', 'mysql', 'mongodb', 'sqlite'}
+
+
+def create_test_tables(config, use_restore_options=False):
+    '''
+    Create test tables for borgmatic to dump and backup.
+    '''
+    command = 'create table test{id} (thing int); insert into test{id} values (1);'
+
+    if 'postgresql_databases' in config:
+        run_postgresql_command(command.format(id=1), config, use_restore_options)
+    if 'mariadb_databases' in config:
+        run_mariadb_command(command.format(id=2), config, use_restore_options)
+    if 'mysql_databases' in config:
+        run_mariadb_command(command.format(id=3), config, use_restore_options, binary_name='mysql')
+    if 'mongodb_databases' in config:
+        get_mongodb_database_client(config, use_restore_options)['test4'].insert_one({'thing': 1})
+    if 'sqlite_databases' in config:
+        run_sqlite_command(command.format(id=5), config, use_restore_options)
+
+
+def drop_test_tables(config, use_restore_options=False):
+    '''
+    Drop the test tables in preparation for borgmatic restoring them.
+    '''
+    command = 'drop table if exists test{id};'
+
+    if 'postgresql_databases' in config:
+        run_postgresql_command(command.format(id=1), config, use_restore_options)
+    if 'mariadb_databases' in config:
+        run_mariadb_command(command.format(id=2), config, use_restore_options)
+    if 'mysql_databases' in config:
+        run_mariadb_command(command.format(id=3), config, use_restore_options, binary_name='mysql')
+    if 'mongodb_databases' in config:
+        get_mongodb_database_client(config, use_restore_options)['test4'].drop()
+    if 'sqlite_databases' in config:
+        run_sqlite_command(command.format(id=5), config, use_restore_options)
+
+
+def select_test_tables(config, use_restore_options=False):
+    '''
+    Select the test tables to make sure they exist.
+
+    Raise if the expected tables cannot be selected, for instance if a restore hasn't worked as
+    expected.
+    '''
+    command = 'select count(*) from test{id};'
+
+    if 'postgresql_databases' in config:
+        run_postgresql_command(command.format(id=1), config, use_restore_options)
+    if 'mariadb_databases' in config:
+        run_mariadb_command(command.format(id=2), config, use_restore_options)
+    if 'mysql_databases' in config:
+        run_mariadb_command(command.format(id=3), config, use_restore_options, binary_name='mysql')
+    if 'mongodb_databases' in config:
+        assert (
+            get_mongodb_database_client(config, use_restore_options)['test4'].count_documents(
+                filter={}
+            )
+            > 0
+        )
+    if 'sqlite_databases' in config:
+        run_sqlite_command(command.format(id=5), config, use_restore_options)
 
 
 def test_database_dump_and_restore():
@@ -188,15 +356,17 @@ def test_database_dump_and_restore():
 
     try:
         config_path = os.path.join(temporary_directory, 'test.yaml')
-        write_configuration(
+        config = write_configuration(
             temporary_directory, config_path, repository_path, borgmatic_source_directory
         )
+        create_test_tables(config)
+        select_test_tables(config)
 
         subprocess.check_call(
             ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
         )
 
-        # Run borgmatic to generate a backup archive including a database dump.
+        # Run borgmatic to generate a backup archive including database dumps.
         subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
 
         # Get the created archive name.
@@ -209,16 +379,21 @@ def test_database_dump_and_restore():
         assert len(parsed_output[0]['archives']) == 1
         archive_name = parsed_output[0]['archives'][0]['archive']
 
-        # Restore the database from the archive.
+        # Restore the databases from the archive.
+        drop_test_tables(config)
         subprocess.check_call(
             ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
         )
+
+        # Ensure the test tables have actually been restored.
+        select_test_tables(config)
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+        drop_test_tables(config)
 
 
-def test_database_dump_and_restore_with_restore_cli_arguments():
+def test_database_dump_and_restore_with_restore_cli_flags():
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()
     repository_path = os.path.join(temporary_directory, 'test.borg')
@@ -228,9 +403,11 @@ def test_database_dump_and_restore_with_restore_cli_arguments():
 
     try:
         config_path = os.path.join(temporary_directory, 'test.yaml')
-        write_simple_custom_restore_configuration(
+        config = write_simple_custom_restore_configuration(
             temporary_directory, config_path, repository_path, borgmatic_source_directory
         )
+        create_test_tables(config)
+        select_test_tables(config)
 
         subprocess.check_call(
             ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
@@ -250,6 +427,7 @@ def test_database_dump_and_restore_with_restore_cli_arguments():
         archive_name = parsed_output[0]['archives'][0]['archive']
 
         # Restore the database from the archive.
+        drop_test_tables(config)
         subprocess.check_call(
             [
                 'borgmatic',
@@ -264,15 +442,25 @@ def test_database_dump_and_restore_with_restore_cli_arguments():
                 'postgresql2',
                 '--port',
                 '5433',
-                '--username',
-                'postgres2',
                 '--password',
                 'test2',
             ]
         )
+
+        # Ensure the test tables have actually been restored. But first modify the config to contain
+        # the altered restore values from the borgmatic command above. This ensures that the test
+        # tables are selected from the correct database.
+        database = config['postgresql_databases'][0]
+        database['restore_hostname'] = 'postgresql2'
+        database['restore_port'] = '5433'
+        database['restore_password'] = 'test2'
+
+        select_test_tables(config, use_restore_options=True)
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+        drop_test_tables(config)
+        drop_test_tables(config, use_restore_options=True)
 
 
 def test_database_dump_and_restore_with_restore_configuration_options():
@@ -285,9 +473,11 @@ def test_database_dump_and_restore_with_restore_configuration_options():
 
     try:
         config_path = os.path.join(temporary_directory, 'test.yaml')
-        write_custom_restore_configuration(
+        config = write_custom_restore_configuration(
             temporary_directory, config_path, repository_path, borgmatic_source_directory
         )
+        create_test_tables(config)
+        select_test_tables(config)
 
         subprocess.check_call(
             ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
@@ -307,12 +497,18 @@ def test_database_dump_and_restore_with_restore_configuration_options():
         archive_name = parsed_output[0]['archives'][0]['archive']
 
         # Restore the database from the archive.
+        drop_test_tables(config)
         subprocess.check_call(
             ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
         )
+
+        # Ensure the test tables have actually been restored.
+        select_test_tables(config, use_restore_options=True)
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+        drop_test_tables(config)
+        drop_test_tables(config, use_restore_options=True)
 
 
 def test_database_dump_and_restore_with_directory_format():
@@ -325,7 +521,7 @@ def test_database_dump_and_restore_with_directory_format():
 
     try:
         config_path = os.path.join(temporary_directory, 'test.yaml')
-        write_configuration(
+        config = write_configuration(
             temporary_directory,
             config_path,
             repository_path,
@@ -333,6 +529,8 @@ def test_database_dump_and_restore_with_directory_format():
             postgresql_dump_format='directory',
             mongodb_dump_format='directory',
         )
+        create_test_tables(config)
+        select_test_tables(config)
 
         subprocess.check_call(
             ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
@@ -342,12 +540,17 @@ def test_database_dump_and_restore_with_directory_format():
         subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
 
         # Restore the database from the archive.
+        drop_test_tables(config)
         subprocess.check_call(
             ['borgmatic', '--config', config_path, 'restore', '--archive', 'latest']
         )
+
+        # Ensure the test tables have actually been restored.
+        select_test_tables(config)
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+        drop_test_tables(config)
 
 
 def test_database_dump_with_error_causes_borgmatic_to_exit():

tests/integration/config/test_load.py

@@ -44,6 +44,14 @@ def test_load_configuration_replaces_complex_constants():
     assert module.load_configuration('config.yaml') == {'key': {'subkey': 'value'}}
 
 
+def test_load_configuration_with_only_integer_value_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    config_file = io.StringIO('33')
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+    assert module.load_configuration('config.yaml') == 33
+
+
 def test_load_configuration_inlines_include_relative_to_current_directory():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -124,6 +132,37 @@ def test_load_configuration_raises_if_absolute_include_does_not_exist():
         assert module.load_configuration('config.yaml')
 
 
+def test_load_configuration_inlines_multiple_file_include_as_list():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(True)
+    flexmock(module.os.path).should_receive('exists').never()
+    include1_file = io.StringIO('value1')
+    include1_file.name = '/root/include1.yaml'
+    builtins.should_receive('open').with_args('/root/include1.yaml').and_return(include1_file)
+    include2_file = io.StringIO('value2')
+    include2_file.name = '/root/include2.yaml'
+    builtins.should_receive('open').with_args('/root/include2.yaml').and_return(include2_file)
+    config_file = io.StringIO('key: !include [/root/include1.yaml, /root/include2.yaml]')
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'key': ['value2', 'value1']}
+
+
+def test_load_configuration_include_with_unsupported_filename_type_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(True)
+    flexmock(module.os.path).should_receive('exists').never()
+    config_file = io.StringIO('key: !include {path: /root/include.yaml}')
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
 def test_load_configuration_merges_include():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -149,6 +188,43 @@ def test_load_configuration_merges_include():
     assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
 
 
+def test_load_configuration_merges_multiple_file_include():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include1_file = io.StringIO(
+        '''
+        foo: bar
+        baz: quux
+        original: yes
+        '''
+    )
+    include1_file.name = 'include1.yaml'
+    builtins.should_receive('open').with_args('/tmp/include1.yaml').and_return(include1_file)
+    include2_file = io.StringIO(
+        '''
+        baz: second
+        '''
+    )
+    include2_file.name = 'include2.yaml'
+    builtins.should_receive('open').with_args('/tmp/include2.yaml').and_return(include2_file)
+    config_file = io.StringIO(
+        '''
+        foo: override
+        <<: !include [include1.yaml, include2.yaml]
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {
+        'foo': 'override',
+        'baz': 'second',
+        'original': 'yes',
+    }
+
+
 def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -438,8 +514,9 @@ def test_raise_omit_node_error_raises():
         module.raise_omit_node_error(loader=flexmock(), node=flexmock())
 
 
-def test_filter_omitted_nodes():
-    nodes = [
+def test_filter_omitted_nodes_discards_values_with_omit_tag_and_also_equal_values():
+    nodes = [flexmock(), flexmock()]
+    values = [
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
@@ -448,11 +525,139 @@ def test_filter_omitted_nodes():
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
     ]
 
-    result = module.filter_omitted_nodes(nodes)
+    result = module.filter_omitted_nodes(nodes, values)
 
     assert [item.value for item in result] == ['a', 'c', 'a', 'c']
 
 
+def test_filter_omitted_nodes_keeps_all_values_when_given_only_one_node():
+    nodes = [flexmock()]
+    values = [
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+    ]
+
+    result = module.filter_omitted_nodes(nodes, values)
+
+    assert [item.value for item in result] == ['a', 'b', 'c', 'a', 'b', 'c']
+
+
+def test_merge_values_combines_mapping_values():
+    nodes = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_hourly'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
+                    ),
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='25'
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_nanosecondly'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='1000'
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    values = module.merge_values(nodes)
+
+    assert len(values) == 4
+    assert values[0][0].value == 'keep_hourly'
+    assert values[0][1].value == '24'
+    assert values[1][0].value == 'keep_daily'
+    assert values[1][1].value == '7'
+    assert values[2][0].value == 'keep_daily'
+    assert values[2][1].value == '25'
+    assert values[3][0].value == 'keep_nanosecondly'
+    assert values[3][1].value == '1000'
+
+
+def test_merge_values_combines_sequence_values():
+    nodes = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.SequenceNode(
+                tag='tag:yaml.org,2002:seq',
+                value=[
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='1'),
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='2'),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.SequenceNode(
+                tag='tag:yaml.org,2002:seq',
+                value=[
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='3'),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.SequenceNode(
+                tag='tag:yaml.org,2002:seq',
+                value=[
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='4'),
+                ],
+            ),
+        ),
+    ]
+
+    values = module.merge_values(nodes)
+
+    assert len(values) == 4
+    assert values[0].value == '1'
+    assert values[1].value == '2'
+    assert values[2].value == '3'
+    assert values[3].value == '4'
+
+
 def test_deep_merge_nodes_replaces_colliding_scalar_values():
     node_values = [
         (
@@ -499,10 +704,10 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values():
     assert section_key.value == 'retention'
     options = section_value.value
     assert len(options) == 2
-    assert options[0][0].value == 'keep_hourly'
-    assert options[0][1].value == '24'
-    assert options[1][0].value == 'keep_daily'
-    assert options[1][1].value == '5'
+    assert options[0][0].value == 'keep_daily'
+    assert options[0][1].value == '5'
+    assert options[1][0].value == 'keep_hourly'
+    assert options[1][1].value == '24'
 
 
 def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
@@ -553,10 +758,10 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
     assert section_key.value == 'retention'
     options = section_value.value
     assert len(options) == 3
-    assert options[0][0].value == 'keep_hourly'
-    assert options[0][1].value == '24'
-    assert options[1][0].value == 'keep_daily'
-    assert options[1][1].value == '7'
+    assert options[0][0].value == 'keep_daily'
+    assert options[0][1].value == '7'
+    assert options[1][0].value == 'keep_hourly'
+    assert options[1][1].value == '24'
     assert options[2][0].value == 'keep_minutely'
     assert options[2][1].value == '10'
 
@@ -629,10 +834,10 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
     assert section_key.value == 'storage'
     options = section_value.value
     assert len(options) == 2
-    assert options[0][0].value == 'lock_wait'
-    assert options[0][1].value == '5'
-    assert options[1][0].value == 'extra_borg_options'
-    nested_options = options[1][1].value
+    assert options[0][0].value == 'extra_borg_options'
+    assert options[1][0].value == 'lock_wait'
+    assert options[1][1].value == '5'
+    nested_options = options[0][1].value
     assert len(nested_options) == 2
     assert nested_options[0][0].value == 'init'
     assert nested_options[0][1].value == '--init-option'

tests/integration/config/test_validate.py

@@ -117,7 +117,7 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
     module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
 
 
-def test_parse_configuration_inlines_include():
+def test_parse_configuration_inlines_include_inside_deprecated_section():
     mock_config_and_schema(
         '''
         source_directories:

tests/unit/actions/test_create.py

@@ -40,6 +40,41 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
     )
 
 
+def test_run_create_with_store_config_files_false_does_not_create_borgmatic_manifest():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
+    flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
+    flexmock(module).should_receive('create_borgmatic_manifest').never()
+    flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({})
+    flexmock(module.borgmatic.hooks.dispatch).should_receive(
+        'call_hooks_even_if_unconfigured'
+    ).and_return({})
+    create_arguments = flexmock(
+        repository=None,
+        progress=flexmock(),
+        stats=flexmock(),
+        json=flexmock(),
+        list_files=flexmock(),
+    )
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
+
+    list(
+        module.run_create(
+            config_filename='test.yaml',
+            repository={'path': 'repo'},
+            config={'store_config_files': False},
+            hook_context={},
+            local_borg_version=None,
+            create_arguments=create_arguments,
+            global_arguments=global_arguments,
+            dry_run_label='',
+            local_path=None,
+            remote_path=None,
+        )
+    )
+
+
 def test_run_create_runs_with_selected_repository():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive(

tests/unit/actions/test_export_key.py

@@ -0,0 +1,20 @@
+from flexmock import flexmock
+
+from borgmatic.actions import export_key as module
+
+
+def test_run_export_key_does_not_raise():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.borg.export_key).should_receive('export_key')
+    export_arguments = flexmock(repository=flexmock())
+
+    module.run_export_key(
+        repository={'path': 'repo'},
+        config={},
+        local_borg_version=None,
+        export_arguments=export_arguments,
+        global_arguments=flexmock(),
+        local_path=None,
+        remote_path=None,
+    )

tests/unit/actions/test_restore.py

@@ -16,6 +16,15 @@ def test_get_configured_database_matches_database_by_name():
     ) == ('postgresql_databases', {'name': 'bar'})
 
 
+def test_get_configured_database_matches_nothing_when_nothing_configured():
+    assert module.get_configured_database(
+        config={},
+        archive_database_names={'postgresql_databases': ['foo']},
+        hook_name='postgresql_databases',
+        database_name='quux',
+    ) == (None, None)
+
+
 def test_get_configured_database_matches_nothing_when_database_name_not_configured():
     assert module.get_configured_database(
         config={'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]},

tests/unit/borg/test_create.py

@@ -647,6 +647,53 @@ def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sourc
     )
 
 
+def test_create_archive_with_sources_and_used_config_paths_with_store_config_files_false_calls_borg_with_sources_and_no_config_paths():
+    flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
+    flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
+    flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('expand_directories').with_args(('foo', 'bar')).and_return(
+        ('foo', 'bar')
+    )
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('pattern_root_directories').and_return([])
+    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
+    flexmock(module).should_receive('expand_home_directories').and_return(())
+    flexmock(module).should_receive('write_pattern_file').and_return(None)
+    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module).should_receive('ensure_files_readable')
+    flexmock(module).should_receive('make_pattern_flags').and_return(())
+    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{DEFAULT_ARCHIVE_NAME}',)
+    )
+    environment = {'BORG_THINGY': 'YUP'}
+    flexmock(module.environment).should_receive('make_environment').and_return(environment)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=environment,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository_path='repo',
+        config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'store_config_files': False,
+        },
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']),
+    )
+
+
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
@@ -713,7 +760,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--info',),
+        ('borg', 'create', '--info') + REPO_ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
@@ -756,7 +803,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
+        ('borg', 'create', '--json') + REPO_ARCHIVE_WITH_PATHS,
         working_directory=None,
         extra_environment=None,
         borg_local_path='borg',
@@ -798,7 +845,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--debug', '--show-rc'),
+        ('borg', 'create', '--debug', '--show-rc') + REPO_ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
@@ -841,7 +888,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
+        ('borg', 'create', '--json') + REPO_ARCHIVE_WITH_PATHS,
         working_directory=None,
         extra_environment=None,
         borg_local_path='borg',
@@ -927,7 +974,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create', '--dry-run') + REPO_ARCHIVE_WITH_PATHS + ('--info',),
+        ('borg', 'create', '--dry-run', '--info') + REPO_ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
         borg_local_path='borg',
@@ -1802,7 +1849,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--stats',),
+        ('borg', 'create', '--stats') + REPO_ARCHIVE_WITH_PATHS,
         output_log_level=module.borgmatic.logger.ANSWER,
         output_file=None,
         borg_local_path='borg',
@@ -1888,12 +1935,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create')
-        + REPO_ARCHIVE_WITH_PATHS
-        + (
-            '--info',
-            '--progress',
-        ),
+        ('borg', 'create', '--info', '--progress') + REPO_ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
         borg_local_path='borg',
@@ -1937,7 +1979,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--progress',),
+        ('borg', 'create', '--progress') + REPO_ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
         borg_local_path='borg',
@@ -1982,10 +2024,12 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('collect_special_file_paths').and_return(())
     create_command = (
-        ('borg', 'create', '--one-file-system', '--read-special')
-        + REPO_ARCHIVE_WITH_PATHS
-        + ('--progress',)
-    )
+        'borg',
+        'create',
+        '--one-file-system',
+        '--read-special',
+        '--progress',
+    ) + REPO_ARCHIVE_WITH_PATHS
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         create_command + ('--dry-run', '--list'),
         processes=processes,
@@ -2111,14 +2155,14 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
-    create_command = (
+    create_flags = (
         'borg',
         'create',
         '--one-file-system',
         '--read-special',
-    ) + REPO_ARCHIVE_WITH_PATHS
+    )
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        create_command + ('--dry-run', '--list'),
+        create_flags + ('--dry-run', '--list') + REPO_ARCHIVE_WITH_PATHS,
         processes=processes,
         output_log_level=logging.INFO,
         output_file=None,
@@ -2127,7 +2171,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
         extra_environment=None,
     )
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        create_command + ('--exclude-from', '/excludes'),
+        create_flags + ('--exclude-from', '/excludes') + REPO_ARCHIVE_WITH_PATHS,
         processes=processes,
         output_log_level=logging.INFO,
         output_file=None,
@@ -2235,7 +2279,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
+        ('borg', 'create', '--json') + REPO_ARCHIVE_WITH_PATHS,
         working_directory=None,
         extra_environment=None,
         borg_local_path='borg',
@@ -2278,7 +2322,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
     )
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('execute_command_and_capture_output').with_args(
-        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
+        ('borg', 'create', '--json') + REPO_ARCHIVE_WITH_PATHS,
         working_directory=None,
         extra_environment=None,
         borg_local_path='borg',

tests/unit/borg/test_environment.py

@@ -19,28 +19,36 @@ def test_make_environment_with_ssh_command_should_set_environment():
     assert environment.get('BORG_RSH') == 'ssh -C'
 
 
-def test_make_environment_without_configuration_should_only_set_default_environment():
+def test_make_environment_without_configuration_should_not_set_environment():
     environment = module.make_environment({})
 
-    assert environment == {
-        'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
-        'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
-        'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING': 'NO',
-    }
+    assert environment == {}
 
 
-def test_make_environment_with_relocated_repo_access_should_override_default():
+def test_make_environment_with_relocated_repo_access_true_should_set_environment_yes():
     environment = module.make_environment({'relocated_repo_access_is_ok': True})
 
     assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
 
 
-def test_make_environment_check_i_know_what_i_am_doing_should_override_default():
+def test_make_environment_with_relocated_repo_access_false_should_set_environment_no():
+    environment = module.make_environment({'relocated_repo_access_is_ok': False})
+
+    assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'no'
+
+
+def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environment_YES():
     environment = module.make_environment({'check_i_know_what_i_am_doing': True})
 
     assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'YES'
 
 
+def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environment_NO():
+    environment = module.make_environment({'check_i_know_what_i_am_doing': False})
+
+    assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'NO'
+
+
 def test_make_environment_with_integer_variable_value():
     environment = module.make_environment({'borg_files_cache_ttl': 40})
     assert environment.get('BORG_FILES_CACHE_TTL') == '40'

tests/unit/borg/test_export_key.py

@@ -0,0 +1,222 @@
+import logging
+
+import pytest
+from flexmock import flexmock
+
+import borgmatic.logger
+from borgmatic.borg import export_key as module
+
+from ..test_verbosity import insert_logging_mock
+
+
+def insert_execute_command_mock(command, output_file=module.DO_NOT_CAPTURE):
+    borgmatic.logger.add_custom_log_levels()
+
+    flexmock(module.environment).should_receive('make_environment')
+    flexmock(module).should_receive('execute_command').with_args(
+        command,
+        output_file=output_file,
+        output_log_level=module.logging.ANSWER,
+        borg_local_path='borg',
+        extra_environment=None,
+    ).once()
+
+
+def test_export_key_calls_borg_with_required_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_calls_borg_with_remote_path_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--remote-path', 'borg1', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+        remote_path='borg1',
+    )
+
+
+def test_export_key_calls_borg_with_umask_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--umask', '0770', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={'umask': '0770'},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_calls_borg_with_log_json_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--log-json', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=True),
+    )
+
+
+def test_export_key_calls_borg_with_lock_wait_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--lock-wait', '5', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={'lock_wait': '5'},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_with_log_info_calls_borg_with_info_parameter():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--info', 'repo'))
+    insert_logging_mock(logging.INFO)
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_with_log_debug_calls_borg_with_debug_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--debug', '--show-rc', 'repo'))
+    insert_logging_mock(logging.DEBUG)
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_calls_borg_with_paper_flags():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--paper', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=True, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_calls_borg_with_paper_flag():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--paper', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=True, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_calls_borg_with_qr_html_flag():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', '--qr-html', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=True, path=None),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_calls_borg_with_path_argument():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    insert_execute_command_mock(('borg', 'key', 'export', 'repo', 'dest'), output_file=None)
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path='dest'),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_with_already_existent_path_raises():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module).should_receive('execute_command').never()
+
+    with pytest.raises(FileExistsError):
+        module.export_key(
+            repository_path='repo',
+            config={},
+            local_borg_version='1.2.3',
+            export_arguments=flexmock(paper=False, qr_html=False, path='dest'),
+            global_arguments=flexmock(dry_run=False, log_json=False),
+        )
+
+
+def test_export_key_with_stdout_path_calls_borg_without_path_argument():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    insert_execute_command_mock(('borg', 'key', 'export', 'repo'))
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path='-'),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+    )
+
+
+def test_export_key_with_dry_run_skip_borg_call():
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+    flexmock(module.os.path).should_receive('exists').never()
+    flexmock(module).should_receive('execute_command').never()
+
+    module.export_key(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        export_arguments=flexmock(paper=False, qr_html=False, path=None),
+        global_arguments=flexmock(dry_run=True, log_json=False),
+    )

tests/unit/borg/test_flags.py

@@ -86,28 +86,28 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a
 
 
 @pytest.mark.parametrize(
-    'match_archives, archive_name_format,feature_available,expected_result',
+    'match_archives,archive_name_format,feature_available,expected_result',
     (
         (None, None, True, ()),
         (None, '', True, ()),
         (
             're:foo-.*',
-            '{hostname}-{now}',
+            '{hostname}-{now}',  # noqa: FS003
             True,
             ('--match-archives', 're:foo-.*'),
-        ),  # noqa: FS003
+        ),
         (
             'sh:foo-*',
-            '{hostname}-{now}',
+            '{hostname}-{now}',  # noqa: FS003
             False,
             ('--glob-archives', 'foo-*'),
-        ),  # noqa: FS003
+        ),
         (
             'foo-*',
-            '{hostname}-{now}',
+            '{hostname}-{now}',  # noqa: FS003
             False,
             ('--glob-archives', 'foo-*'),
-        ),  # noqa: FS003
+        ),
         (
             None,
             '{hostname}-docs-{now}',  # noqa: FS003
@@ -133,6 +133,18 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a
             False,
             ('--glob-archives', '{hostname}-docs-*'),  # noqa: FS003
         ),
+        (
+            None,
+            '{now}',  # noqa: FS003
+            False,
+            (),
+        ),
+        (
+            None,
+            '{now}',  # noqa: FS003
+            True,
+            (),
+        ),
         (None, '{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')),  # noqa: FS003
     ),
 )

tests/unit/borg/test_prune.py

@@ -118,6 +118,19 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead():
     assert result == expected
 
 
+def test_make_prune_flags_ignores_keep_exclude_tags_in_config():
+    config = {
+        'keep_daily': 1,
+        'keep_exclude_tags': True,
+    }
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
+
+    result = module.make_prune_flags(config, local_borg_version='1.2.3')
+
+    assert result == ('--keep-daily', '1')
+
+
 PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3')
 
 

tests/unit/commands/test_borgmatic.py

@@ -748,6 +748,24 @@ def test_run_actions_runs_break_lock():
     )
 
 
+def test_run_actions_runs_export_key():
+    flexmock(module).should_receive('add_custom_log_levels')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(borgmatic.actions.export_key).should_receive('run_export_key').once()
+
+    tuple(
+        module.run_actions(
+            arguments={'global': flexmock(dry_run=False, log_file='foo'), 'export': flexmock()},
+            config_filename=flexmock(),
+            config={'repositories': []},
+            local_path=flexmock(),
+            remote_path=flexmock(),
+            local_borg_version=flexmock(),
+            repository={'path': 'repo'},
+        )
+    )
+
+
 def test_run_actions_runs_borg():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module.command).should_receive('execute_hook')

tests/unit/config/test_load.py

@@ -0,0 +1,37 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_probe_and_include_file_with_absolute_path_skips_probing():
+    config = flexmock()
+    flexmock(module).should_receive('load_configuration').with_args('/etc/include.yaml').and_return(
+        config
+    ).once()
+
+    assert module.probe_and_include_file('/etc/include.yaml', ['/etc', '/var']) == config
+
+
+def test_probe_and_include_file_with_relative_path_probes_include_directories():
+    config = flexmock()
+    flexmock(module.os.path).should_receive('exists').with_args('/etc/include.yaml').and_return(
+        False
+    )
+    flexmock(module.os.path).should_receive('exists').with_args('/var/include.yaml').and_return(
+        True
+    )
+    flexmock(module).should_receive('load_configuration').with_args('/etc/include.yaml').never()
+    flexmock(module).should_receive('load_configuration').with_args('/var/include.yaml').and_return(
+        config
+    ).once()
+
+    assert module.probe_and_include_file('include.yaml', ['/etc', '/var']) == config
+
+
+def test_probe_and_include_file_with_relative_path_and_missing_files_raises():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module).should_receive('load_configuration').never()
+
+    with pytest.raises(FileNotFoundError):
+        module.probe_and_include_file('include.yaml', ['/etc', '/var'])

tests/unit/config/test_normalize.py

@@ -111,6 +111,13 @@ def test_normalize_sections_with_different_umask_values_raises():
         module.normalize_sections('test.yaml', config)
 
 
+def test_normalize_sections_with_only_scalar_raises():
+    config = 33
+
+    with pytest.raises(ValueError):
+        module.normalize_sections('test.yaml', config)
+
+
 @pytest.mark.parametrize(
     'config,expected_config,produces_logs',
     (

tests/unit/hooks/test_mariadb.py

@@ -0,0 +1,621 @@
+import logging
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks import mariadb as module
+
+
+def test_database_names_to_dump_passes_through_name():
+    extra_environment = flexmock()
+    log_prefix = ''
+
+    names = module.database_names_to_dump(
+        {'name': 'foo'}, extra_environment, log_prefix, dry_run=False
+    )
+
+    assert names == ('foo',)
+
+
+def test_database_names_to_dump_bails_for_dry_run():
+    extra_environment = flexmock()
+    log_prefix = ''
+    flexmock(module).should_receive('execute_command_and_capture_output').never()
+
+    names = module.database_names_to_dump(
+        {'name': 'all'}, extra_environment, log_prefix, dry_run=True
+    )
+
+    assert names == ()
+
+
+def test_database_names_to_dump_queries_mariadb_for_database_names():
+    extra_environment = flexmock()
+    log_prefix = ''
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'),
+        extra_environment=extra_environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all'}, extra_environment, log_prefix, dry_run=False
+    )
+
+    assert names == ('foo', 'bar')
+
+
+def test_dump_databases_dumps_each_database():
+    databases = [{'name': 'foo'}, {'name': 'bar'}]
+    processes = [flexmock(), flexmock()]
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
+
+    for name, process in zip(('foo', 'bar'), processes):
+        flexmock(module).should_receive('execute_dump_command').with_args(
+            database={'name': name},
+            log_prefix=object,
+            dump_path=object,
+            database_names=(name,),
+            extra_environment=object,
+            dry_run=object,
+            dry_run_label=object,
+        ).and_return(process).once()
+
+    assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes
+
+
+def test_dump_databases_dumps_with_password():
+    database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'}
+    process = flexmock()
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
+
+    flexmock(module).should_receive('execute_dump_command').with_args(
+        database=database,
+        log_prefix=object,
+        dump_path=object,
+        database_names=('foo',),
+        extra_environment={'MYSQL_PWD': 'trustsome1'},
+        dry_run=object,
+        dry_run_label=object,
+    ).and_return(process).once()
+
+    assert module.dump_databases([database], {}, 'test.yaml', dry_run=False) == [process]
+
+
+def test_dump_databases_dumps_all_databases_at_once():
+    databases = [{'name': 'all'}]
+    process = flexmock()
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('execute_dump_command').with_args(
+        database={'name': 'all'},
+        log_prefix=object,
+        dump_path=object,
+        database_names=('foo', 'bar'),
+        extra_environment=object,
+        dry_run=object,
+        dry_run_label=object,
+    ).and_return(process).once()
+
+    assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == [process]
+
+
+def test_dump_databases_dumps_all_databases_separately_when_format_configured():
+    databases = [{'name': 'all', 'format': 'sql'}]
+    processes = [flexmock(), flexmock()]
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
+
+    for name, process in zip(('foo', 'bar'), processes):
+        flexmock(module).should_receive('execute_dump_command').with_args(
+            database={'name': name, 'format': 'sql'},
+            log_prefix=object,
+            dump_path=object,
+            database_names=(name,),
+            extra_environment=object,
+            dry_run=object,
+            dry_run_label=object,
+        ).and_return(process).once()
+
+    assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False) == processes
+
+
+def test_database_names_to_dump_runs_mariadb_with_list_options():
+    database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf'}
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        (
+            'mariadb',
+            '--defaults-extra-file=mariadb.cnf',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+        extra_environment=None,
+    ).and_return(('foo\nbar')).once()
+
+    assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
+
+
+def test_execute_dump_command_runs_mariadb_dump():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mariadb-dump',
+            '--add-drop-database',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo'},
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
+def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mariadb-dump',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'add_drop_database': False},
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
+def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mariadb-dump',
+            '--add-drop-database',
+            '--host',
+            'database.example.org',
+            '--port',
+            '5433',
+            '--protocol',
+            'tcp',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433},
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
+def test_execute_dump_command_runs_mariadb_dump_with_username_and_password():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mariadb-dump',
+            '--add-drop-database',
+            '--user',
+            'root',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        extra_environment={'MYSQL_PWD': 'trustsome1'},
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'},
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment={'MYSQL_PWD': 'trustsome1'},
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
+def test_execute_dump_command_runs_mariadb_dump_with_options():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mariadb-dump',
+            '--stuff=such',
+            '--add-drop-database',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'options': '--stuff=such'},
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
+def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump():
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
+    flexmock(module).should_receive('execute_command').never()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo'},
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=True,
+            dry_run_label='SO DRY',
+        )
+        is None
+    )
+
+
+def test_execute_dump_command_with_dry_run_skips_mariadb_dump():
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').never()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo'},
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=True,
+            dry_run_label='SO DRY',
+        )
+        is None
+    )
+
+
+def test_dump_databases_errors_for_missing_all_databases():
+    databases = [{'name': 'all'}]
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/all'
+    )
+    flexmock(module).should_receive('database_names_to_dump').and_return(())
+
+    with pytest.raises(ValueError):
+        assert module.dump_databases(databases, {}, 'test.yaml', dry_run=False)
+
+
+def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run():
+    databases = [{'name': 'all'}]
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/all'
+    )
+    flexmock(module).should_receive('database_names_to_dump').and_return(())
+
+    assert module.dump_databases(databases, {}, 'test.yaml', dry_run=True) == []
+
+
+def test_restore_database_dump_runs_mariadb_to_restore():
+    hook_config = [{'name': 'foo'}, {'name': 'bar'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mariadb', '--batch'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        database={'name': 'foo'},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_runs_mariadb_with_options():
+    hook_config = [{'name': 'foo', 'restore_options': '--harder'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mariadb', '--batch', '--harder'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        database=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_runs_mariadb_with_hostname_and_port():
+    hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mariadb',
+            '--batch',
+            '--host',
+            'database.example.org',
+            '--port',
+            '5433',
+            '--protocol',
+            'tcp',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        database=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_runs_mariadb_with_username_and_password():
+    hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mariadb', '--batch', '--user', 'root'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'MYSQL_PWD': 'trustsome1'},
+    ).once()
+
+    module.restore_database_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        database=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
+    hook_config = [
+        {
+            'name': 'foo',
+            'username': 'root',
+            'password': 'trustsome1',
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+            'restore_username': 'restoreusername',
+            'restore_password': 'restorepassword',
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mariadb',
+            '--batch',
+            '--host',
+            'clihost',
+            '--port',
+            'cliport',
+            '--protocol',
+            'tcp',
+            '--user',
+            'cliusername',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'MYSQL_PWD': 'clipassword'},
+    ).once()
+
+    module.restore_database_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        database=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': 'clihost',
+            'port': 'cliport',
+            'username': 'cliusername',
+            'password': 'clipassword',
+        },
+    )
+
+
+def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
+    hook_config = [
+        {
+            'name': 'foo',
+            'username': 'root',
+            'password': 'trustsome1',
+            'hostname': 'dbhost',
+            'port': 'dbport',
+            'restore_username': 'restoreuser',
+            'restore_password': 'restorepass',
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mariadb',
+            '--batch',
+            '--host',
+            'restorehost',
+            '--port',
+            'restoreport',
+            '--protocol',
+            'tcp',
+            '--user',
+            'restoreuser',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'MYSQL_PWD': 'restorepass'},
+    ).once()
+
+    module.restore_database_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        database=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_with_dry_run_skips_restore():
+    hook_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+
+    module.restore_database_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        database={'name': 'foo'},
+        dry_run=True,
+        extract_process=flexmock(),
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )

tests/unit/hooks/test_mongodb.py

@@ -1,6 +1,5 @@
 import logging
 
-import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks import mongodb as module
@@ -17,7 +16,7 @@ def test_dump_databases_runs_mongodump_for_each_database():
 
     for name, process in zip(('foo', 'bar'), processes):
         flexmock(module).should_receive('execute_command').with_args(
-            ['mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'],
+            ('mongodump', '--db', name, '--archive', '>', f'databases/localhost/{name}'),
             shell=True,
             run_to_completion=False,
         ).and_return(process).once()
@@ -47,7 +46,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port():
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        [
+        (
             'mongodump',
             '--host',
             'database.example.org',
@@ -58,7 +57,7 @@ def test_dump_databases_runs_mongodump_with_hostname_and_port():
             '--archive',
             '>',
             'databases/database.example.org/foo',
-        ],
+        ),
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
@@ -83,7 +82,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password():
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        [
+        (
             'mongodump',
             '--username',
             'mongo',
@@ -96,7 +95,7 @@ def test_dump_databases_runs_mongodump_with_username_and_password():
             '--archive',
             '>',
             'databases/localhost/foo',
-        ],
+        ),
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
@@ -114,7 +113,7 @@ def test_dump_databases_runs_mongodump_with_directory_format():
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
 
     flexmock(module).should_receive('execute_command').with_args(
-        ['mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'],
+        ('mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'),
         shell=True,
     ).and_return(flexmock()).once()
 
@@ -131,7 +130,15 @@ def test_dump_databases_runs_mongodump_with_options():
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ['mongodump', '--db', 'foo', '--stuff=such', '--archive', '>', 'databases/localhost/foo'],
+        (
+            'mongodump',
+            '--db',
+            'foo',
+            '--stuff=such',
+            '--archive',
+            '>',
+            'databases/localhost/foo',
+        ),
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
@@ -149,7 +156,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases():
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ['mongodump', '--archive', '>', 'databases/localhost/all'],
+        ('mongodump', '--archive', '>', 'databases/localhost/all'),
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
@@ -158,23 +165,23 @@ def test_dump_databases_runs_mongodumpall_for_all_databases():
 
 
 def test_restore_database_dump_runs_mongorestore():
-    databases_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
+    hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ['mongorestore', '--archive', '--drop', '--db', 'foo'],
+        ['mongorestore', '--archive', '--drop'],
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -186,33 +193,8 @@ def test_restore_database_dump_runs_mongorestore():
     )
 
 
-def test_restore_database_dump_errors_on_empty_databases_config():
-    databases_config = []
-
-    flexmock(module).should_receive('make_dump_path')
-    flexmock(module.dump).should_receive('make_database_dump_filename')
-    flexmock(module).should_receive('execute_command_with_processes').never()
-    flexmock(module).should_receive('execute_command').never()
-
-    with pytest.raises(ValueError):
-        module.restore_database_dump(
-            databases_config,
-            {},
-            'test.yaml',
-            database_name='foo',
-            dry_run=False,
-            extract_process=flexmock(),
-            connection_params={
-                'hostname': None,
-                'port': None,
-                'username': None,
-                'password': None,
-            },
-        )
-
-
 def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
-    databases_config = [
+    hook_config = [
         {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None}
     ]
     extract_process = flexmock(stdout=flexmock())
@@ -224,8 +206,6 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
             'mongorestore',
             '--archive',
             '--drop',
-            '--db',
-            'foo',
             '--host',
             'database.example.org',
             '--port',
@@ -237,10 +217,10 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -253,7 +233,7 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
 
 
 def test_restore_database_dump_runs_mongorestore_with_username_and_password():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'username': 'mongo',
@@ -271,8 +251,6 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
             'mongorestore',
             '--archive',
             '--drop',
-            '--db',
-            'foo',
             '--username',
             'mongo',
             '--password',
@@ -286,10 +264,10 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -302,7 +280,7 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
 
 
 def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'username': 'mongo',
@@ -324,8 +302,6 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
             'mongorestore',
             '--archive',
             '--drop',
-            '--db',
-            'foo',
             '--host',
             'clihost',
             '--port',
@@ -343,10 +319,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -359,7 +335,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
 
 
 def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'username': 'mongo',
@@ -381,8 +357,6 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
             'mongorestore',
             '--archive',
             '--drop',
-            '--db',
-            'foo',
             '--host',
             'restorehost',
             '--port',
@@ -400,10 +374,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -416,23 +390,23 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
 
 
 def test_restore_database_dump_runs_mongorestore_with_options():
-    databases_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}]
+    hook_config = [{'name': 'foo', 'restore_options': '--harder', 'schemas': None}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ['mongorestore', '--archive', '--drop', '--db', 'foo', '--harder'],
+        ['mongorestore', '--archive', '--drop', '--harder'],
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -445,7 +419,7 @@ def test_restore_database_dump_runs_mongorestore_with_options():
 
 
 def test_restore_databases_dump_runs_mongorestore_with_schemas():
-    databases_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
+    hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('make_dump_path')
@@ -455,8 +429,6 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
             'mongorestore',
             '--archive',
             '--drop',
-            '--db',
-            'foo',
             '--nsInclude',
             'bar',
             '--nsInclude',
@@ -468,10 +440,10 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -484,7 +456,7 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
 
 
 def test_restore_database_dump_runs_psql_for_all_database_dump():
-    databases_config = [{'name': 'all', 'schemas': None}]
+    hook_config = [{'name': 'all', 'schemas': None}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('make_dump_path')
@@ -497,10 +469,10 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='all',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -513,17 +485,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
 
 
 def test_restore_database_dump_with_dry_run_skips_restore():
-    databases_config = [{'name': 'foo', 'schemas': None}]
+    hook_config = [{'name': 'foo', 'schemas': None}]
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=True,
         extract_process=flexmock(),
         connection_params={
@@ -536,22 +508,22 @@ def test_restore_database_dump_with_dry_run_skips_restore():
 
 
 def test_restore_database_dump_without_extract_process_restores_from_disk():
-    databases_config = [{'name': 'foo', 'format': 'directory', 'schemas': None}]
+    hook_config = [{'name': 'foo', 'format': 'directory', 'schemas': None}]
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ['mongorestore', '--dir', '/dump/path', '--drop', '--db', 'foo'],
+        ['mongorestore', '--dir', '/dump/path', '--drop'],
         processes=[],
         output_log_level=logging.DEBUG,
         input_file=None,
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=False,
         extract_process=None,
         connection_params={

tests/unit/hooks/test_mysql.py

@@ -380,7 +380,7 @@ def test_dump_databases_does_not_error_for_missing_all_databases_with_dry_run():
 
 
 def test_restore_database_dump_runs_mysql_to_restore():
-    databases_config = [{'name': 'foo'}, {'name': 'bar'}]
+    hook_config = [{'name': 'foo'}, {'name': 'bar'}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').with_args(
@@ -392,10 +392,10 @@ def test_restore_database_dump_runs_mysql_to_restore():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -407,31 +407,8 @@ def test_restore_database_dump_runs_mysql_to_restore():
     )
 
 
-def test_restore_database_dump_errors_when_database_missing_from_configuration():
-    databases_config = [{'name': 'foo'}, {'name': 'bar'}]
-    extract_process = flexmock(stdout=flexmock())
-
-    flexmock(module).should_receive('execute_command_with_processes').never()
-
-    with pytest.raises(ValueError):
-        module.restore_database_dump(
-            databases_config,
-            {},
-            'test.yaml',
-            database_name='other',
-            dry_run=False,
-            extract_process=extract_process,
-            connection_params={
-                'hostname': None,
-                'port': None,
-                'username': None,
-                'password': None,
-            },
-        )
-
-
 def test_restore_database_dump_runs_mysql_with_options():
-    databases_config = [{'name': 'foo', 'restore_options': '--harder'}]
+    hook_config = [{'name': 'foo', 'restore_options': '--harder'}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').with_args(
@@ -443,10 +420,10 @@ def test_restore_database_dump_runs_mysql_with_options():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -459,7 +436,7 @@ def test_restore_database_dump_runs_mysql_with_options():
 
 
 def test_restore_database_dump_runs_mysql_with_hostname_and_port():
-    databases_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').with_args(
@@ -480,10 +457,10 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -496,7 +473,7 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
 
 
 def test_restore_database_dump_runs_mysql_with_username_and_password():
-    databases_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
+    hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').with_args(
@@ -508,10 +485,10 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -524,7 +501,7 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
 
 
 def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'username': 'root',
@@ -557,10 +534,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -573,7 +550,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
 
 
 def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'username': 'root',
@@ -608,10 +585,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -624,15 +601,15 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
 
 
 def test_restore_database_dump_with_dry_run_skips_restore():
-    databases_config = [{'name': 'foo'}]
+    hook_config = [{'name': 'foo'}]
 
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=True,
         extract_process=flexmock(),
         connection_params={

tests/unit/hooks/test_ntfy.py

@@ -10,8 +10,8 @@ custom_base_url = 'https://ntfy.example.com'
 topic = 'borgmatic-unit-testing'
 
 custom_message_config = {
-    'title': 'Borgmatic unit testing',
-    'message': 'Borgmatic unit testing',
+    'title': 'borgmatic unit testing',
+    'message': 'borgmatic unit testing',
     'priority': 'min',
     'tags': '+1',
 }
@@ -26,8 +26,8 @@ custom_message_headers = {
 
 def return_default_message_headers(state=Enum):
     headers = {
-        'X-Title': f'A Borgmatic {state.name} event happened',
-        'X-Message': f'A Borgmatic {state.name} event happened',
+        'X-Title': f'A borgmatic {state.name} event happened',
+        'X-Message': f'A borgmatic {state.name} event happened',
         'X-Priority': 'default',
         'X-Tags': 'borgmatic',
     }

tests/unit/hooks/test_postgresql.py

@@ -464,7 +464,7 @@ def test_dump_databases_runs_non_default_pg_dump():
 
 
 def test_restore_database_dump_runs_pg_restore():
-    databases_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
+    hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
@@ -500,10 +500,10 @@ def test_restore_database_dump_runs_pg_restore():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -515,32 +515,8 @@ def test_restore_database_dump_runs_pg_restore():
     )
 
 
-def test_restore_database_dump_errors_when_database_missing_from_configuration():
-    databases_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
-    extract_process = flexmock(stdout=flexmock())
-
-    flexmock(module).should_receive('execute_command_with_processes').never()
-    flexmock(module).should_receive('execute_command').never()
-
-    with pytest.raises(ValueError):
-        module.restore_database_dump(
-            databases_config,
-            {},
-            'test.yaml',
-            database_name='other',
-            dry_run=False,
-            extract_process=extract_process,
-            connection_params={
-                'hostname': None,
-                'port': None,
-                'username': None,
-                'password': None,
-            },
-        )
-
-
 def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
-    databases_config = [
+    hook_config = [
         {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None}
     ]
     extract_process = flexmock(stdout=flexmock())
@@ -586,10 +562,10 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -602,7 +578,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
 
 
 def test_restore_database_dump_runs_pg_restore_with_username_and_password():
-    databases_config = [
+    hook_config = [
         {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None}
     ]
     extract_process = flexmock(stdout=flexmock())
@@ -646,10 +622,10 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -662,7 +638,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
 
 
 def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'hostname': 'database.example.org',
@@ -725,10 +701,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -741,7 +717,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
 
 
 def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'hostname': 'database.example.org',
@@ -804,10 +780,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -820,7 +796,7 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
 
 
 def test_restore_database_dump_runs_pg_restore_with_options():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'restore_options': '--harder',
@@ -865,10 +841,10 @@ def test_restore_database_dump_runs_pg_restore_with_options():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -881,7 +857,7 @@ def test_restore_database_dump_runs_pg_restore_with_options():
 
 
 def test_restore_database_dump_runs_psql_for_all_database_dump():
-    databases_config = [{'name': 'all', 'schemas': None}]
+    hook_config = [{'name': 'all', 'schemas': None}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
@@ -904,10 +880,10 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='all',
+        database={'name': 'all'},
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -920,7 +896,7 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
 
 
 def test_restore_database_dump_runs_psql_for_plain_database_dump():
-    databases_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}]
+    hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
@@ -948,10 +924,10 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -964,7 +940,7 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
 
 
 def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
-    databases_config = [
+    hook_config = [
         {
             'name': 'foo',
             'pg_restore_command': 'docker exec mycontainer pg_restore',
@@ -1013,10 +989,10 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={
@@ -1029,7 +1005,7 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
 
 
 def test_restore_database_dump_with_dry_run_skips_restore():
-    databases_config = [{'name': 'foo', 'schemas': None}]
+    hook_config = [{'name': 'foo', 'schemas': None}]
 
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('make_dump_path')
@@ -1037,10 +1013,10 @@ def test_restore_database_dump_with_dry_run_skips_restore():
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=True,
         extract_process=flexmock(),
         connection_params={
@@ -1053,7 +1029,7 @@ def test_restore_database_dump_with_dry_run_skips_restore():
 
 
 def test_restore_database_dump_without_extract_process_restores_from_disk():
-    databases_config = [{'name': 'foo', 'schemas': None}]
+    hook_config = [{'name': 'foo', 'schemas': None}]
 
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('make_dump_path')
@@ -1089,10 +1065,10 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database={'name': 'foo'},
         dry_run=False,
         extract_process=None,
         connection_params={
@@ -1105,7 +1081,7 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
 
 
 def test_restore_database_dump_with_schemas_restores_schemas():
-    databases_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
+    hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
 
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('make_dump_path')
@@ -1145,10 +1121,10 @@ def test_restore_database_dump_with_schemas_restores_schemas():
     ).once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='foo',
+        database=hook_config[0],
         dry_run=False,
         extract_process=None,
         connection_params={

tests/unit/hooks/test_sqlite.py

@@ -1,6 +1,5 @@
 import logging
 
-import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks import sqlite as module
@@ -93,7 +92,7 @@ def test_dump_databases_does_not_dump_if_dry_run():
 
 
 def test_restore_database_dump_restores_database():
-    databases_config = [{'path': '/path/to/database', 'name': 'database'}, {'name': 'other'}]
+    hook_config = [{'path': '/path/to/database', 'name': 'database'}, {'name': 'other'}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').with_args(
@@ -109,10 +108,10 @@ def test_restore_database_dump_restores_database():
     flexmock(module.os).should_receive('remove').once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='database',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={'restore_path': None},
@@ -120,7 +119,7 @@ def test_restore_database_dump_restores_database():
 
 
 def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
-    databases_config = [
+    hook_config = [
         {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
     ]
     extract_process = flexmock(stdout=flexmock())
@@ -138,10 +137,10 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
     flexmock(module.os).should_receive('remove').once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='database',
+        database={'name': 'database'},
         dry_run=False,
         extract_process=extract_process,
         connection_params={'restore_path': 'cli/path/to/database'},
@@ -149,7 +148,7 @@ def test_restore_database_dump_with_connection_params_uses_connection_params_for
 
 
 def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
-    databases_config = [
+    hook_config = [
         {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
     ]
     extract_process = flexmock(stdout=flexmock())
@@ -167,10 +166,10 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
     flexmock(module.os).should_receive('remove').once()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='database',
+        database=hook_config[0],
         dry_run=False,
         extract_process=extract_process,
         connection_params={'restore_path': None},
@@ -178,34 +177,18 @@ def test_restore_database_dump_without_connection_params_uses_restore_params_in_
 
 
 def test_restore_database_dump_does_not_restore_database_if_dry_run():
-    databases_config = [{'path': '/path/to/database', 'name': 'database'}]
+    hook_config = [{'path': '/path/to/database', 'name': 'database'}]
     extract_process = flexmock(stdout=flexmock())
 
     flexmock(module).should_receive('execute_command_with_processes').never()
     flexmock(module.os).should_receive('remove').never()
 
     module.restore_database_dump(
-        databases_config,
+        hook_config,
         {},
         'test.yaml',
-        database_name='database',
+        database={'name': 'database'},
         dry_run=True,
         extract_process=extract_process,
         connection_params={'restore_path': None},
     )
-
-
-def test_restore_database_dump_raises_error_if_database_config_is_empty():
-    databases_config = []
-    extract_process = flexmock(stdout=flexmock())
-
-    with pytest.raises(ValueError):
-        module.restore_database_dump(
-            databases_config,
-            {},
-            'test.yaml',
-            database_name='database',
-            dry_run=False,
-            extract_process=extract_process,
-            connection_params={'restore_path': None},
-        )