Bump version for release.

Merge pull request #73 from diivi/feat/restore-with-different-hostname-port-username

.drone.yml

@@ -8,16 +8,38 @@ services:
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
+  - name: postgresql2
+    image: docker.io/postgres:13.1-alpine
+    environment:
+      POSTGRES_PASSWORD: test2
+      POSTGRES_DB: test
+      POSTGRES_USER: postgres2
+    commands:
+      - docker-entrypoint.sh -p 5433
   - name: mysql
     image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
+  - name: mysql2
+    image: docker.io/mariadb:10.5
+    environment:
+      MYSQL_ROOT_PASSWORD: test2
+      MYSQL_DATABASE: test
+    commands:
+      - docker-entrypoint.sh --port=3307
   - name: mongodb
     image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
+  - name: mongodb2
+    image: docker.io/mongo:5.0.5
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root2
+      MONGO_INITDB_ROOT_PASSWORD: test2
+    commands:
+      - docker-entrypoint.sh --port=27018
 
 clone:
   skip_verify: true

NEWS

@@ -1,3 +1,17 @@
+1.7.15
+ * #326: Add configuration options and command-line flags for backing up a database from one
+   location while restoring it somewhere else.
+ * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors.
+ * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style
+   configuration.
+ * #529: Deprecate generate-borgmatic-config in favor of new "config generate" action.
+ * #529: Deprecate validate-borgmatic-config in favor of new "config validate" action.
+ * #697, #712, #716: Extract borgmatic configuration from backup via new "config bootstrap"
+   action—even when borgmatic has no configuration yet!
+ * #669: Add sample systemd user service for running borgmatic as a non-root user.
+ * #711, #713: Fix an error when "data" check time files are accessed without getting upgraded
+   first.
+
 1.7.14
  * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file,
    or monitoring), so not even errors are shown.

README.md

@@ -121,8 +121,10 @@ issues.
 
 ### Social
 
-Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for
-general Borg and borgmatic discussion and support.
+~~Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for
+general Borg and borgmatic discussion and support.~~ borgmatic supports the
+ongoing [Reddit user
+protests](https://www.theverge.com/2023/6/10/23756476/reddit-protest-api-changes-apollo-third-party-apps).
 
 Also follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
 

borgmatic/actions/config/bootstrap.py

@@ -0,0 +1,105 @@
+import json
+import logging
+import os
+
+import borgmatic.borg.extract
+import borgmatic.borg.rlist
+import borgmatic.config.validate
+import borgmatic.hooks.command
+from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+logger = logging.getLogger(__name__)
+
+
+def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Given:
+    The bootstrap arguments, which include the repository and archive name, borgmatic source directory,
+    destination directory, and whether to strip components.
+    The global arguments, which include the dry run flag
+    and the local borg version,
+    Return:
+    The config paths from the manifest.json file in the borgmatic source directory after extracting it from the
+    repository.
+
+    Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
+    expected configuration path data.
+    '''
+    borgmatic_source_directory = (
+        bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+    )
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
+    )
+    extract_process = borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        bootstrap_arguments.repository,
+        borgmatic.borg.rlist.resolve_archive_name(
+            bootstrap_arguments.repository,
+            bootstrap_arguments.archive,
+            {},
+            local_borg_version,
+            global_arguments,
+        ),
+        [borgmatic_manifest_path],
+        {},
+        {},
+        local_borg_version,
+        global_arguments,
+        extract_to_stdout=True,
+    )
+
+    manifest_json = extract_process.stdout.read()
+    if not manifest_json:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to missing bootstrap manifest'
+        )
+
+    try:
+        manifest_data = json.loads(manifest_json)
+    except json.JSONDecodeError as error:
+        raise ValueError(
+            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
+        )
+
+    try:
+        return manifest_data['config_paths']
+    except KeyError:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to invalid bootstrap manifest'
+        )
+
+
+def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Run the "bootstrap" action for the given repository.
+
+    Raise ValueError if the bootstrap configuration could not be loaded.
+    Raise CalledProcessError or OSError if Borg could not be run.
+    '''
+    manifest_config_paths = get_config_paths(
+        bootstrap_arguments, global_arguments, local_borg_version
+    )
+
+    logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
+
+    borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        bootstrap_arguments.repository,
+        borgmatic.borg.rlist.resolve_archive_name(
+            bootstrap_arguments.repository,
+            bootstrap_arguments.archive,
+            {},
+            local_borg_version,
+            global_arguments,
+        ),
+        [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
+        {},
+        {},
+        local_borg_version,
+        global_arguments,
+        extract_to_stdout=False,
+        destination_path=bootstrap_arguments.destination,
+        strip_components=bootstrap_arguments.strip_components,
+        progress=bootstrap_arguments.progress,
+    )

borgmatic/actions/config/generate.py

@@ -0,0 +1,46 @@
+import logging
+
+import borgmatic.config.generate
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_generate(generate_arguments, global_arguments):
+    '''
+    Given the generate arguments and the global arguments, each as an argparse.Namespace instance,
+    run the "generate" action.
+
+    Raise FileExistsError if a file already exists at the destination path and the generate
+    arguments do not have overwrite set.
+    '''
+    dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
+
+    logger.answer(
+        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
+    )
+
+    borgmatic.config.generate.generate_sample_configuration(
+        global_arguments.dry_run,
+        generate_arguments.source_filename,
+        generate_arguments.destination_filename,
+        borgmatic.config.validate.schema_filename(),
+        overwrite=generate_arguments.overwrite,
+    )
+
+    if generate_arguments.source_filename:
+        logger.answer(
+            f'''
+Merged in the contents of configuration file at: {generate_arguments.source_filename}
+To review the changes made, run:
+
+    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
+        )
+
+    logger.answer(
+        '''
+This includes all available configuration options with example values, the few
+required options as indicated. Please edit the file to suit your needs.
+
+If you ever need help: https://torsion.org/borgmatic/#issues'''
+    )

borgmatic/actions/config/validate.py

@@ -0,0 +1,22 @@
+import logging
+
+import borgmatic.config.generate
+
+logger = logging.getLogger(__name__)
+
+
+def run_validate(validate_arguments, configs):
+    '''
+    Given the validate arguments as an argparse.Namespace instance and a dict of configuration
+    filename to corresponding parsed configuration, run the "validate" action.
+
+    Most of the validation is actually performed implicitly by the standard borgmatic configuration
+    loading machinery prior to here, so this function mainly exists to support additional validate
+    flags like "--show".
+    '''
+    if validate_arguments.show:
+        for config_path, config in configs.items():
+            if len(configs) > 1:
+                logger.answer('---')
+
+            logger.answer(borgmatic.config.generate.render_configuration(config))

borgmatic/actions/create.py

@@ -1,7 +1,14 @@
 import json
 import logging
+import os
+
+try:
+    import importlib_metadata
+except ModuleNotFoundError:  # pragma: nocover
+    import importlib.metadata as importlib_metadata
 
 import borgmatic.borg.create
+import borgmatic.borg.state
 import borgmatic.config.validate
 import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
@@ -10,6 +17,35 @@ import borgmatic.hooks.dump
 logger = logging.getLogger(__name__)
 
 
+def create_borgmatic_manifest(location, config_paths, dry_run):
+    '''
+    Create a borgmatic manifest file to store the paths to the configuration files used to create
+    the archive.
+    '''
+    if dry_run:
+        return
+
+    borgmatic_source_directory = location.get(
+        'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+    )
+
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
+    )
+
+    if not os.path.exists(borgmatic_manifest_path):
+        os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
+
+    with open(borgmatic_manifest_path, 'w') as config_list_file:
+        json.dump(
+            {
+                'borgmatic_version': importlib_metadata.version('borgmatic'),
+                'config_paths': config_paths,
+            },
+            config_list_file,
+        )
+
+
 def run_create(
     config_filename,
     repository,
@@ -59,6 +95,9 @@ def run_create(
         location,
         global_arguments.dry_run,
     )
+    create_borgmatic_manifest(
+        location, global_arguments.used_config_paths, global_arguments.dry_run
+    )
     stream_processes = [process for processes in active_dumps.values() for process in processes]
 
     json_output = borgmatic.borg.create.create_archive(

borgmatic/actions/restore.py

@@ -68,9 +68,11 @@ def restore_single_database(
     archive_name,
     hook_name,
     database,
+    connection_params,
 ):  # pragma: no cover
     '''
-    Given (among other things) an archive name, a database hook name, and a configured database
+    Given (among other things) an archive name, a database hook name, the hostname,
+    port, username and password as connection params, and a configured database
     configuration dict, restore that database from the archive.
     '''
     logger.info(
@@ -113,6 +115,7 @@ def restore_single_database(
         location,
         global_arguments.dry_run,
         extract_process,
+        connection_params,
     )
 
 
@@ -301,6 +304,13 @@ def run_restore(
     restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
     found_names = set()
     remaining_restore_names = {}
+    connection_params = {
+        'hostname': restore_arguments.hostname,
+        'port': restore_arguments.port,
+        'username': restore_arguments.username,
+        'password': restore_arguments.password,
+        'restore_path': restore_arguments.restore_path,
+    }
 
     for hook_name, database_names in restore_names.items():
         for database_name in database_names:
@@ -327,6 +337,7 @@ def run_restore(
                 archive_name,
                 found_hook_name or hook_name,
                 dict(found_database, **{'schemas': restore_arguments.schemas}),
+                connection_params,
             )
 
     # For any database that weren't found via exact matches in the hooks configuration, try to
@@ -356,6 +367,7 @@ def run_restore(
                 archive_name,
                 found_hook_name or hook_name,
                 dict(database, **{'schemas': restore_arguments.schemas}),
+                connection_params,
             )
 
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(

borgmatic/borg/borg.py

@@ -38,10 +38,7 @@ def run_arbitrary_borg(
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
 
-        if (
-            borg_command
-            and borg_command[0] in borgmatic.commands.arguments.SUBPARSER_ALIASES.keys()
-        ):
+        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
             logger.warning(
                 f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
             )

borgmatic/borg/check.py

@@ -342,7 +342,7 @@ def upgrade_check_times(location_config, borg_repository_id):
         temporary_path = f'{old_path}.temp'
 
         if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
-            return
+            continue
 
         logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
 

borgmatic/borg/create.py

@@ -351,7 +351,9 @@ def create_archive(
     sources = deduplicate_directories(
         map_directories_to_devices(
             expand_directories(
-                tuple(location_config.get('source_directories', ())) + borgmatic_source_directories
+                tuple(location_config.get('source_directories', ()))
+                + borgmatic_source_directories
+                + tuple(global_arguments.used_config_paths)
             )
         ),
         additional_directory_devices=map_directories_to_devices(

borgmatic/commands/arguments.py

@@ -1,14 +1,17 @@
 import collections
+import itertools
+import sys
 from argparse import Action, ArgumentParser
 
 from borgmatic.config import collect
 
-SUBPARSER_ALIASES = {
+ACTION_ALIASES = {
     'rcreate': ['init', '-I'],
     'prune': ['-p'],
     'compact': [],
     'create': ['-C'],
     'check': ['-k'],
+    'config': [],
     'extract': ['-x'],
     'export-tar': [],
     'mount': ['-m'],
@@ -24,80 +27,209 @@ SUBPARSER_ALIASES = {
 }
 
 
-def parse_subparser_arguments(unparsed_arguments, subparsers):
+def get_subaction_parsers(action_parser):
     '''
-    Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser
-    instance, give each requested action's subparser a shot at parsing all arguments. This allows
-    common arguments like "--repository" to be shared across multiple subparsers.
+    Given an argparse.ArgumentParser instance, lookup the subactions in it and return a dict from
+    subaction name to subaction parser.
+    '''
+    if not action_parser._subparsers:
+        return {}
 
-    Return the result as a tuple of (a dict mapping from subparser name to a parsed namespace of
-    arguments, a list of remaining arguments not claimed by any subparser).
-    '''
-    arguments = collections.OrderedDict()
-    remaining_arguments = list(unparsed_arguments)
-    alias_to_subparser_name = {
-        alias: subparser_name
-        for subparser_name, aliases in SUBPARSER_ALIASES.items()
-        for alias in aliases
+    return {
+        subaction_name: subaction_parser
+        for group_action in action_parser._subparsers._group_actions
+        for subaction_name, subaction_parser in group_action.choices.items()
     }
 
-    # If the "borg" action is used, skip all other subparsers. This avoids confusion like
-    # "borg list" triggering borgmatic's own list action.
-    if 'borg' in unparsed_arguments:
-        subparsers = {'borg': subparsers['borg']}
 
-    for argument in remaining_arguments:
-        canonical_name = alias_to_subparser_name.get(argument, argument)
-        subparser = subparsers.get(canonical_name)
+def get_subactions_for_actions(action_parsers):
+    '''
+    Given a dict from action name to an argparse.ArgumentParser instance, make a map from action
+    name to the names of contained sub-actions.
+    '''
+    return {
+        action: tuple(
+            subaction_name
+            for group_action in action_parser._subparsers._group_actions
+            for subaction_name in group_action.choices.keys()
+        )
+        for action, action_parser in action_parsers.items()
+        if action_parser._subparsers
+    }
 
-        if not subparser:
-            continue
 
-        # If a parsed value happens to be the same as the name of a subparser, remove it from the
-        # remaining arguments. This prevents, for instance, "check --only extract" from triggering
-        # the "extract" subparser.
-        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments):
+    '''
+    Given a sequence of string arguments and a dict from action name to parsed argparse.Namespace
+    arguments, return the string arguments with any values omitted that happen to be the same as
+    the name of a borgmatic action.
+
+    This prevents, for instance, "check --only extract" from triggering the "extract" action.
+    '''
+    remaining_arguments = list(unparsed_arguments)
+
+    for action_name, parsed in parsed_arguments.items():
         for value in vars(parsed).values():
             if isinstance(value, str):
-                if value in subparsers:
+                if value in ACTION_ALIASES.keys():
                     remaining_arguments.remove(value)
             elif isinstance(value, list):
                 for item in value:
-                    if item in subparsers:
+                    if item in ACTION_ALIASES.keys():
                         remaining_arguments.remove(item)
 
-        arguments[canonical_name] = parsed
+    return tuple(remaining_arguments)
 
-    # If no actions are explicitly requested, assume defaults.
-    if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
-        for subparser_name in ('create', 'prune', 'compact', 'check'):
-            subparser = subparsers[subparser_name]
-            parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
-            arguments[subparser_name] = parsed
 
-    remaining_arguments = list(unparsed_arguments)
+def parse_and_record_action_arguments(
+    unparsed_arguments, parsed_arguments, action_parser, action_name, canonical_name=None
+):
+    '''
+    Given unparsed arguments as a sequence of strings, parsed arguments as a dict from action name
+    to parsed argparse.Namespace, a parser to parse with, an action name, and an optional canonical
+    action name (in case this the action name is an alias), parse the arguments and return a list of
+    any remaining string arguments that were not parsed. Also record the parsed argparse.Namespace
+    by setting it into the given parsed arguments. Return None if no parsing occurs because the
+    given action doesn't apply to the given unparsed arguments.
+    '''
+    filtered_arguments = omit_values_colliding_with_action_names(
+        unparsed_arguments, parsed_arguments
+    )
 
-    # Now ask each subparser, one by one, to greedily consume arguments.
-    for subparser_name, subparser in subparsers.items():
-        if subparser_name not in arguments.keys():
+    if action_name not in filtered_arguments:
+        return tuple(unparsed_arguments)
+
+    parsed, remaining = action_parser.parse_known_args(filtered_arguments)
+    parsed_arguments[canonical_name or action_name] = parsed
+
+    # Special case: If this is a "borg" action, greedily consume all arguments after (+1) the "borg"
+    # argument.
+    if action_name == 'borg':
+        borg_options_index = remaining.index('borg') + 1
+        parsed_arguments['borg'].options = remaining[borg_options_index:]
+        remaining = remaining[:borg_options_index]
+
+    return tuple(argument for argument in remaining if argument != action_name)
+
+
+def get_unparsable_arguments(remaining_action_arguments):
+    '''
+    Given a sequence of argument tuples (one per action parser that parsed arguments), determine the
+    remaining arguments that no action parsers have consumed.
+    '''
+    if not remaining_action_arguments:
+        return ()
+
+    return tuple(
+        argument
+        for argument in dict.fromkeys(
+            itertools.chain.from_iterable(remaining_action_arguments)
+        ).keys()
+        if all(argument in action_arguments for action_arguments in remaining_action_arguments)
+    )
+
+
+def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parser):
+    '''
+    Given a sequence of arguments, a dict from action name to argparse.ArgumentParser instance,
+    and the global parser as a argparse.ArgumentParser instance, give each requested action's
+    parser a shot at parsing all arguments. This allows common arguments like "--repository" to be
+    shared across multiple action parsers.
+
+    Return the result as a tuple of: (a dict mapping from action name to an argparse.Namespace of
+    parsed arguments, a tuple of argument tuples where each is the remaining arguments not claimed
+    by any action parser).
+    '''
+    arguments = collections.OrderedDict()
+    help_requested = bool('--help' in unparsed_arguments or '-h' in unparsed_arguments)
+    remaining_action_arguments = []
+    alias_to_action_name = {
+        alias: action_name for action_name, aliases in ACTION_ALIASES.items() for alias in aliases
+    }
+
+    # If the "borg" action is used, skip all other action parsers. This avoids confusion like
+    # "borg list" triggering borgmatic's own list action.
+    if 'borg' in unparsed_arguments:
+        action_parsers = {'borg': action_parsers['borg']}
+
+    # Ask each action parser, one by one, to parse arguments.
+    for argument in unparsed_arguments:
+        action_name = argument
+        canonical_name = alias_to_action_name.get(action_name, action_name)
+        action_parser = action_parsers.get(action_name)
+
+        if not action_parser:
             continue
 
-        subparser = subparsers[subparser_name]
-        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+        subaction_parsers = get_subaction_parsers(action_parser)
 
-    # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the
-    # "borg" action.
-    if 'borg' in arguments:
-        borg_options_index = remaining_arguments.index('borg') + 1
-        arguments['borg'].options = remaining_arguments[borg_options_index:]
-        remaining_arguments = remaining_arguments[:borg_options_index]
+        # But first parse with subaction parsers, if any.
+        if subaction_parsers:
+            subactions_parsed = False
 
-    # Remove the subparser names themselves.
-    for subparser_name, subparser in subparsers.items():
-        if subparser_name in remaining_arguments:
-            remaining_arguments.remove(subparser_name)
+            for subaction_name, subaction_parser in subaction_parsers.items():
+                remaining_action_arguments.append(
+                    tuple(
+                        argument
+                        for argument in parse_and_record_action_arguments(
+                            unparsed_arguments,
+                            arguments,
+                            subaction_parser,
+                            subaction_name,
+                        )
+                        if argument != action_name
+                    )
+                )
 
-    return (arguments, remaining_arguments)
+                if subaction_name in arguments:
+                    subactions_parsed = True
+
+            if not subactions_parsed:
+                if help_requested:
+                    action_parser.print_help()
+                    sys.exit(0)
+                else:
+                    raise ValueError(
+                        f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}"
+                    )
+        # Otherwise, parse with the main action parser.
+        else:
+            remaining_action_arguments.append(
+                parse_and_record_action_arguments(
+                    unparsed_arguments, arguments, action_parser, action_name, canonical_name
+                )
+            )
+
+    # If no actions were explicitly requested, assume defaults.
+    if not arguments and not help_requested:
+        for default_action_name in ('create', 'prune', 'compact', 'check'):
+            default_action_parser = action_parsers[default_action_name]
+            remaining_action_arguments.append(
+                parse_and_record_action_arguments(
+                    tuple(unparsed_arguments) + (default_action_name,),
+                    arguments,
+                    default_action_parser,
+                    default_action_name,
+                )
+            )
+
+    arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments)
+    remaining_action_arguments.append(remaining)
+
+    # Prevent action names that follow "--config" paths from being considered as additional paths.
+    for argument_name in arguments.keys():
+        if argument_name == 'global':
+            continue
+
+        for action_name in [argument_name] + ACTION_ALIASES.get(argument_name, []):
+            if action_name in arguments['global'].config_paths:
+                arguments['global'].config_paths.remove(action_name)
+                break
+
+    return (
+        arguments,
+        tuple(remaining_action_arguments) if arguments else unparsed_arguments,
+    )
 
 
 class Extend_action(Action):
@@ -109,14 +241,17 @@ class Extend_action(Action):
         items = getattr(namespace, self.dest, None)
 
         if items:
-            items.extend(values)
+            items.extend(values)  # pragma: no cover
         else:
             setattr(namespace, self.dest, list(values))
 
 
 def make_parsers():
     '''
-    Build a top-level parser and its subparsers and return them as a tuple.
+    Build a global arguments parser, individual action parsers, and a combined parser containing
+    both. Return them as a tuple. The global parser is useful for parsing just global arguments
+    while ignoring actions, and the combined parser is handy for displaying help that includes
+    everything: global flags, a list of actions, etc.
     '''
     config_paths = collect.get_default_config_paths(expand_home=True)
     unexpanded_config_paths = collect.get_default_config_paths(expand_home=False)
@@ -226,7 +361,7 @@ def make_parsers():
         help='Display installed version number of borgmatic and exit',
     )
 
-    top_level_parser = ArgumentParser(
+    global_plus_action_parser = ArgumentParser(
         description='''
             Simple, configuration-driven backup software for servers and workstations. If none of
             the action options are given, then borgmatic defaults to: create, prune, compact, and
@@ -235,14 +370,14 @@ def make_parsers():
         parents=[global_parser],
     )
 
-    subparsers = top_level_parser.add_subparsers(
+    action_parsers = global_plus_action_parser.add_subparsers(
         title='actions',
         metavar='',
         help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:',
     )
-    rcreate_parser = subparsers.add_parser(
+    rcreate_parser = action_parsers.add_parser(
         'rcreate',
-        aliases=SUBPARSER_ALIASES['rcreate'],
+        aliases=ACTION_ALIASES['rcreate'],
         help='Create a new, empty Borg repository',
         description='Create a new, empty Borg repository',
         add_help=False,
@@ -288,9 +423,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    transfer_parser = subparsers.add_parser(
+    transfer_parser = action_parsers.add_parser(
         'transfer',
-        aliases=SUBPARSER_ALIASES['transfer'],
+        aliases=ACTION_ALIASES['transfer'],
         help='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]',
         description='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]',
         add_help=False,
@@ -361,9 +496,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    prune_parser = subparsers.add_parser(
+    prune_parser = action_parsers.add_parser(
         'prune',
-        aliases=SUBPARSER_ALIASES['prune'],
+        aliases=ACTION_ALIASES['prune'],
         help='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)',
         description='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)',
         add_help=False,
@@ -405,9 +540,9 @@ def make_parsers():
     )
     prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    compact_parser = subparsers.add_parser(
+    compact_parser = action_parsers.add_parser(
         'compact',
-        aliases=SUBPARSER_ALIASES['compact'],
+        aliases=ACTION_ALIASES['compact'],
         help='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]',
         description='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]',
         add_help=False,
@@ -441,9 +576,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    create_parser = subparsers.add_parser(
+    create_parser = action_parsers.add_parser(
         'create',
-        aliases=SUBPARSER_ALIASES['create'],
+        aliases=ACTION_ALIASES['create'],
         help='Create an archive (actually perform a backup)',
         description='Create an archive (actually perform a backup)',
         add_help=False,
@@ -475,9 +610,9 @@ def make_parsers():
     )
     create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    check_parser = subparsers.add_parser(
+    check_parser = action_parsers.add_parser(
         'check',
-        aliases=SUBPARSER_ALIASES['check'],
+        aliases=ACTION_ALIASES['check'],
         help='Check archives for consistency',
         description='Check archives for consistency',
         add_help=False,
@@ -517,9 +652,9 @@ def make_parsers():
     )
     check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    extract_parser = subparsers.add_parser(
+    extract_parser = action_parsers.add_parser(
         'extract',
-        aliases=SUBPARSER_ALIASES['extract'],
+        aliases=ACTION_ALIASES['extract'],
         help='Extract files from a named archive to the current directory',
         description='Extract a named archive to the current directory',
         add_help=False,
@@ -563,9 +698,118 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    export_tar_parser = subparsers.add_parser(
+    config_parser = action_parsers.add_parser(
+        'config',
+        aliases=ACTION_ALIASES['config'],
+        help='Perform configuration file related operations',
+        description='Perform configuration file related operations',
+        add_help=False,
+    )
+
+    config_group = config_parser.add_argument_group('config arguments')
+    config_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    config_parsers = config_parser.add_subparsers(
+        title='config sub-actions',
+    )
+
+    config_bootstrap_parser = config_parsers.add_parser(
+        'bootstrap',
+        help='Extract the borgmatic configuration files from a named archive',
+        description='Extract the borgmatic configuration files from a named archive',
+        add_help=False,
+    )
+    config_bootstrap_group = config_bootstrap_parser.add_argument_group(
+        'config bootstrap arguments'
+    )
+    config_bootstrap_group.add_argument(
+        '--repository',
+        help='Path of repository to extract config files from',
+        required=True,
+    )
+    config_bootstrap_group.add_argument(
+        '--borgmatic-source-directory',
+        help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic',
+    )
+    config_bootstrap_group.add_argument(
+        '--archive',
+        help='Name of archive to extract config files from, defaults to "latest"',
+        default='latest',
+    )
+    config_bootstrap_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Directory to extract config files into, defaults to /',
+        default='/',
+    )
+    config_bootstrap_group.add_argument(
+        '--strip-components',
+        type=lambda number: number if number == 'all' else int(number),
+        metavar='NUMBER',
+        help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements',
+    )
+    config_bootstrap_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is extracted',
+    )
+    config_bootstrap_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    config_generate_parser = config_parsers.add_parser(
+        'generate',
+        help='Generate a sample borgmatic configuration file',
+        description='Generate a sample borgmatic configuration file',
+        add_help=False,
+    )
+    config_generate_group = config_generate_parser.add_argument_group('config generate arguments')
+    config_generate_group.add_argument(
+        '-s',
+        '--source',
+        dest='source_filename',
+        help='Optional configuration file to merge into the generated configuration, useful for upgrading your configuration',
+    )
+    config_generate_group.add_argument(
+        '-d',
+        '--destination',
+        dest='destination_filename',
+        default=config_paths[0],
+        help=f'Destination configuration file, default: {unexpanded_config_paths[0]}',
+    )
+    config_generate_group.add_argument(
+        '--overwrite',
+        default=False,
+        action='store_true',
+        help='Whether to overwrite any existing destination file, defaults to false',
+    )
+    config_generate_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    config_validate_parser = config_parsers.add_parser(
+        'validate',
+        help='Validate borgmatic configuration files specified with --config (see borgmatic --help)',
+        description='Validate borgmatic configuration files specified with --config (see borgmatic --help)',
+        add_help=False,
+    )
+    config_validate_group = config_validate_parser.add_argument_group('config validate arguments')
+    config_validate_group.add_argument(
+        '-s',
+        '--show',
+        action='store_true',
+        help='Show the validated configuration after all include merging has occurred',
+    )
+    config_validate_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    export_tar_parser = action_parsers.add_parser(
         'export-tar',
-        aliases=SUBPARSER_ALIASES['export-tar'],
+        aliases=ACTION_ALIASES['export-tar'],
         help='Export an archive to a tar-formatted file or stream',
         description='Export an archive to a tar-formatted file or stream',
         add_help=False,
@@ -609,9 +853,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    mount_parser = subparsers.add_parser(
+    mount_parser = action_parsers.add_parser(
         'mount',
-        aliases=SUBPARSER_ALIASES['mount'],
+        aliases=ACTION_ALIASES['mount'],
         help='Mount files from a named archive as a FUSE filesystem',
         description='Mount a named archive as a FUSE filesystem',
         add_help=False,
@@ -674,9 +918,9 @@ def make_parsers():
     mount_group.add_argument('--options', dest='options', help='Extra Borg mount options')
     mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    umount_parser = subparsers.add_parser(
+    umount_parser = action_parsers.add_parser(
         'umount',
-        aliases=SUBPARSER_ALIASES['umount'],
+        aliases=ACTION_ALIASES['umount'],
         help='Unmount a FUSE filesystem that was mounted with "borgmatic mount"',
         description='Unmount a mounted FUSE filesystem',
         add_help=False,
@@ -691,9 +935,9 @@ def make_parsers():
     )
     umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    restore_parser = subparsers.add_parser(
+    restore_parser = action_parsers.add_parser(
         'restore',
-        aliases=SUBPARSER_ALIASES['restore'],
+        aliases=ACTION_ALIASES['restore'],
         help='Restore database dumps from a named archive',
         description='Restore database dumps from a named archive. (To extract files instead, use "borgmatic extract".)',
         add_help=False,
@@ -720,13 +964,33 @@ def make_parsers():
         dest='schemas',
         help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
     )
+    restore_group.add_argument(
+        '--hostname',
+        help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--port',
+        help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--username',
+        help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--password',
+        help='Password with which to connect to the restore database. Defaults to the "restore_password" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--restore-path',
+        help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
+    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    rlist_parser = subparsers.add_parser(
+    rlist_parser = action_parsers.add_parser(
         'rlist',
-        aliases=SUBPARSER_ALIASES['rlist'],
+        aliases=ACTION_ALIASES['rlist'],
         help='List repository',
         description='List the archives in a repository',
         add_help=False,
@@ -784,9 +1048,9 @@ def make_parsers():
     )
     rlist_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    list_parser = subparsers.add_parser(
+    list_parser = action_parsers.add_parser(
         'list',
-        aliases=SUBPARSER_ALIASES['list'],
+        aliases=ACTION_ALIASES['list'],
         help='List archive',
         description='List the files in an archive or search for a file across archives',
         add_help=False,
@@ -857,9 +1121,9 @@ def make_parsers():
     )
     list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    rinfo_parser = subparsers.add_parser(
+    rinfo_parser = action_parsers.add_parser(
         'rinfo',
-        aliases=SUBPARSER_ALIASES['rinfo'],
+        aliases=ACTION_ALIASES['rinfo'],
         help='Show repository summary information such as disk space used',
         description='Show repository summary information such as disk space used',
         add_help=False,
@@ -874,9 +1138,9 @@ def make_parsers():
     )
     rinfo_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    info_parser = subparsers.add_parser(
+    info_parser = action_parsers.add_parser(
         'info',
-        aliases=SUBPARSER_ALIASES['info'],
+        aliases=ACTION_ALIASES['info'],
         help='Show archive summary information such as disk space used',
         description='Show archive summary information such as disk space used',
         add_help=False,
@@ -935,9 +1199,9 @@ def make_parsers():
     )
     info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    break_lock_parser = subparsers.add_parser(
+    break_lock_parser = action_parsers.add_parser(
         'break-lock',
-        aliases=SUBPARSER_ALIASES['break-lock'],
+        aliases=ACTION_ALIASES['break-lock'],
         help='Break the repository and cache locks left behind by Borg aborting',
         description='Break Borg repository and cache locks left behind by Borg aborting',
         add_help=False,
@@ -951,9 +1215,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    borg_parser = subparsers.add_parser(
+    borg_parser = action_parsers.add_parser(
         'borg',
-        aliases=SUBPARSER_ALIASES['borg'],
+        aliases=ACTION_ALIASES['borg'],
         help='Run an arbitrary Borg command',
         description="Run an arbitrary Borg command based on borgmatic's configuration",
         add_help=False,
@@ -973,20 +1237,41 @@ def make_parsers():
     )
     borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    return top_level_parser, subparsers
+    return global_parser, action_parsers, global_plus_action_parser
 
 
 def parse_arguments(*unparsed_arguments):
     '''
     Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
-    '''
-    top_level_parser, subparsers = make_parsers()
+    them as a dict mapping from action name (or "global") to an argparse.Namespace instance.
 
-    arguments, remaining_arguments = parse_subparser_arguments(
-        unparsed_arguments, subparsers.choices
+    Raise ValueError if the arguments cannot be parsed.
+    Raise SystemExit with an error code of 0 if "--help" was requested.
+    '''
+    global_parser, action_parsers, global_plus_action_parser = make_parsers()
+    arguments, remaining_action_arguments = parse_arguments_for_actions(
+        unparsed_arguments, action_parsers.choices, global_parser
     )
-    arguments['global'] = top_level_parser.parse_args(remaining_arguments)
+
+    for action_name in ('bootstrap', 'generate', 'validate'):
+        if (
+            action_name in arguments.keys() and len(arguments.keys()) > 2
+        ):  # 2 = 1 for 'global' + 1 for the action
+            raise ValueError(
+                f'The {action_name} action cannot be combined with other actions. Please run it separately.'
+            )
+
+    unknown_arguments = get_unparsable_arguments(remaining_action_arguments)
+
+    if unknown_arguments:
+        if '--help' in unknown_arguments or '-h' in unknown_arguments:
+            global_plus_action_parser.print_help()
+            sys.exit(0)
+
+        global_plus_action_parser.print_usage()
+        raise ValueError(
+            f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}"
+        )
 
     if arguments['global'].excludes_filename:
         raise ValueError(

borgmatic/commands/borgmatic.py

@@ -18,6 +18,9 @@ import borgmatic.actions.borg
 import borgmatic.actions.break_lock
 import borgmatic.actions.check
 import borgmatic.actions.compact
+import borgmatic.actions.config.bootstrap
+import borgmatic.actions.config.generate
+import borgmatic.actions.config.validate
 import borgmatic.actions.create
 import borgmatic.actions.export_tar
 import borgmatic.actions.extract
@@ -30,11 +33,12 @@ import borgmatic.actions.restore
 import borgmatic.actions.rinfo
 import borgmatic.actions.rlist
 import borgmatic.actions.transfer
-import borgmatic.commands.completion
+import borgmatic.commands.completion.bash
+import borgmatic.commands.completion.fish
 from borgmatic.borg import umount as borg_umount
 from borgmatic.borg import version as borg_version
 from borgmatic.commands.arguments import parse_arguments
-from borgmatic.config import checks, collect, convert, validate
+from borgmatic.config import checks, collect, validate
 from borgmatic.hooks import command, dispatch, monitor
 from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
@@ -497,6 +501,9 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
     and sequence of logging.LogRecord instances containing any parse errors.
+
+    Log records are returned here instead of being logged directly because logging isn't yet
+    initialized at this point!
     '''
     # Dict mapping from config filename to corresponding parsed config dict.
     configs = collections.OrderedDict()
@@ -504,6 +511,17 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
 
     # Parse and load each configuration file.
     for config_filename in config_filenames:
+        logs.extend(
+            [
+                logging.makeLogRecord(
+                    dict(
+                        levelno=logging.DEBUG,
+                        levelname='DEBUG',
+                        msg=f'{config_filename}: Loading configuration file',
+                    )
+                ),
+            ]
+        )
         try:
             configs[config_filename], parse_logs = validate.parse_configuration(
                 config_filename, validate.schema_filename(), overrides, resolve_env
@@ -600,9 +618,101 @@ def get_local_path(configs):
     return next(iter(configs.values())).get('location', {}).get('local_path', 'borg')
 
 
+def collect_highlander_action_summary_logs(configs, arguments, configuration_parse_errors):
+    '''
+    Given a dict of configuration filename to corresponding parsed configuration, parsed
+    command-line arguments as a dict from subparser name to a parsed namespace of arguments, and
+    whether any configuration files encountered errors during parsing, run a highlander action
+    specified in the arguments, if any, and yield a series of logging.LogRecord instances containing
+    summary information.
+
+    A highlander action is an action that cannot coexist with other actions on the borgmatic
+    command-line, and borgmatic exits after processing such an action.
+    '''
+    if 'bootstrap' in arguments:
+        try:
+            # No configuration file is needed for bootstrap.
+            local_borg_version = borg_version.local_borg_version({}, 'borg')
+        except (OSError, CalledProcessError, ValueError) as error:
+            yield from log_error_records('Error getting local Borg version', error)
+            return
+
+        try:
+            borgmatic.actions.config.bootstrap.run_bootstrap(
+                arguments['bootstrap'], arguments['global'], local_borg_version
+            )
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.ANSWER,
+                    levelname='ANSWER',
+                    msg='Bootstrap successful',
+                )
+            )
+        except (
+            CalledProcessError,
+            ValueError,
+            OSError,
+        ) as error:
+            yield from log_error_records(error)
+
+        return
+
+    if 'generate' in arguments:
+        try:
+            borgmatic.actions.config.generate.run_generate(
+                arguments['generate'], arguments['global']
+            )
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.ANSWER,
+                    levelname='ANSWER',
+                    msg='Generate successful',
+                )
+            )
+        except (
+            CalledProcessError,
+            ValueError,
+            OSError,
+        ) as error:
+            yield from log_error_records(error)
+
+        return
+
+    if 'validate' in arguments:
+        if configuration_parse_errors:
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.CRITICAL,
+                    levelname='CRITICAL',
+                    msg='Configuration validation failed',
+                )
+            )
+
+            return
+
+        try:
+            borgmatic.actions.config.validate.run_validate(arguments['validate'], configs)
+
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.ANSWER,
+                    levelname='ANSWER',
+                    msg='All configuration files are valid',
+                )
+            )
+        except (
+            CalledProcessError,
+            ValueError,
+            OSError,
+        ) as error:
+            yield from log_error_records(error)
+
+        return
+
+
 def collect_configuration_run_summary_logs(configs, arguments):
     '''
-    Given a dict of configuration filename to corresponding parsed configuration, and parsed
+    Given a dict of configuration filename to corresponding parsed configuration and parsed
     command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
     each configuration file and yield a series of logging.LogRecord instances containing summary
     information about each run.
@@ -705,7 +815,7 @@ def exit_with_help_link():  # pragma: no cover
     sys.exit(1)
 
 
-def main():  # pragma: no cover
+def main(extra_summary_logs=[]):  # pragma: no cover
     configure_signals()
 
     try:
@@ -726,16 +836,20 @@ def main():  # pragma: no cover
         print(importlib_metadata.version('borgmatic'))
         sys.exit(0)
     if global_arguments.bash_completion:
-        print(borgmatic.commands.completion.bash_completion())
+        print(borgmatic.commands.completion.bash.bash_completion())
         sys.exit(0)
     if global_arguments.fish_completion:
-        print(borgmatic.commands.completion.fish_completion())
+        print(borgmatic.commands.completion.fish.fish_completion())
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
+    global_arguments.used_config_paths = list(config_filenames)
     configs, parse_logs = load_configurations(
         config_filenames, global_arguments.overrides, global_arguments.resolve_env
     )
+    configuration_parse_errors = (
+        (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False
+    )
 
     any_json_flags = any(
         getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
@@ -758,10 +872,18 @@ def main():  # pragma: no cover
         logger.critical(f'Error configuring logging: {error}')
         exit_with_help_link()
 
-    logger.debug('Ensuring legacy configuration is upgraded')
-    convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
-
-    summary_logs = parse_logs + list(collect_configuration_run_summary_logs(configs, arguments))
+    summary_logs = (
+        extra_summary_logs
+        + parse_logs
+        + (
+            list(
+                collect_highlander_action_summary_logs(
+                    configs, arguments, configuration_parse_errors
+                )
+            )
+            or list(collect_configuration_run_summary_logs(configs, arguments))
+        )
+    )
     summary_logs_max_level = max(log.levelno for log in summary_logs)
 
     for message in ('', 'summary:'):

borgmatic/commands/completion/actions.py

@@ -0,0 +1,36 @@
+import borgmatic.commands.arguments
+
+
+def upgrade_message(language: str, upgrade_command: str, completion_file: str):
+    return f'''
+Your {language} completions script is from a different version of borgmatic than is
+currently installed. Please upgrade your script so your completions match the
+command-line flags in your installed borgmatic! Try this to upgrade:
+
+    {upgrade_command}
+    source {completion_file}
+'''
+
+
+def available_actions(subparsers, current_action=None):
+    '''
+    Given subparsers as an argparse._SubParsersAction instance and a current action name (if
+    any), return the actions names that can follow the current action on a command-line.
+
+    This takes into account which sub-actions that the current action supports. For instance, if
+    "bootstrap" is a sub-action for "config", then "bootstrap" should be able to follow a current
+    action of "config" but not "list".
+    '''
+    action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions(
+        subparsers.choices
+    )
+    current_subactions = action_to_subactions.get(current_action)
+
+    if current_subactions:
+        return current_subactions
+
+    all_subactions = set(
+        subaction for subactions in action_to_subactions.values() for subaction in subactions
+    )
+
+    return tuple(action for action in subparsers.choices.keys() if action not in all_subactions)

borgmatic/commands/completion/bash.py

@@ -0,0 +1,66 @@
+import borgmatic.commands.arguments
+import borgmatic.commands.completion.actions
+
+
+def parser_flags(parser):
+    '''
+    Given an argparse.ArgumentParser instance, return its argument flags in a space-separated
+    string.
+    '''
+    return ' '.join(option for action in parser._actions for option in action.option_strings)
+
+
+def bash_completion():
+    '''
+    Return a bash completion script for the borgmatic command. Produce this by introspecting
+    borgmatic's command-line argument parsers.
+    '''
+    (
+        unused_global_parser,
+        action_parsers,
+        global_plus_action_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
+    global_flags = parser_flags(global_plus_action_parser)
+
+    # Avert your eyes.
+    return '\n'.join(
+        (
+            'check_version() {',
+            '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
+            '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
+            '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
+            f'''        then cat << EOF\n{borgmatic.commands.completion.actions.upgrade_message(
+                    'bash',
+                    'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
+                    '$BASH_SOURCE',
+                )}\nEOF''',
+            '    fi',
+            '}',
+            'complete_borgmatic() {',
+        )
+        + tuple(
+            '''    if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
+        COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
+        return 0
+    fi'''
+            % (
+                action,
+                parser_flags(action_parser),
+                ' '.join(
+                    borgmatic.commands.completion.actions.available_actions(action_parsers, action)
+                ),
+                global_flags,
+            )
+            for action, action_parser in reversed(action_parsers.choices.items())
+        )
+        + (
+            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
+            % (
+                ' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)),
+                global_flags,
+            ),
+            '    (check_version &)',
+            '}',
+            '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
+        )
+    )

borgmatic/commands/completion/fish.py

@@ -2,72 +2,8 @@ import shlex
 from argparse import Action
 from textwrap import dedent
 
-from borgmatic.commands import arguments
-
-
-def upgrade_message(language: str, upgrade_command: str, completion_file: str):
-    return f'''
-Your {language} completions script is from a different version of borgmatic than is
-currently installed. Please upgrade your script so your completions match the
-command-line flags in your installed borgmatic! Try this to upgrade:
-
-    {upgrade_command}
-    source {completion_file}
-'''
-
-
-def parser_flags(parser):
-    '''
-    Given an argparse.ArgumentParser instance, return its argument flags in a space-separated
-    string.
-    '''
-    return ' '.join(option for action in parser._actions for option in action.option_strings)
-
-
-def bash_completion():
-    '''
-    Return a bash completion script for the borgmatic command. Produce this by introspecting
-    borgmatic's command-line argument parsers.
-    '''
-    top_level_parser, subparsers = arguments.make_parsers()
-    global_flags = parser_flags(top_level_parser)
-    actions = ' '.join(subparsers.choices.keys())
-
-    # Avert your eyes.
-    return '\n'.join(
-        (
-            'check_version() {',
-            '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
-            '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
-            '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
-            f'''        then cat << EOF\n{upgrade_message(
-                    'bash',
-                    'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
-                    '$BASH_SOURCE',
-                )}\nEOF''',
-            '    fi',
-            '}',
-            'complete_borgmatic() {',
-        )
-        + tuple(
-            '''    if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
-        COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
-        return 0
-    fi'''
-            % (action, parser_flags(subparser), actions, global_flags)
-            for action, subparser in subparsers.choices.items()
-        )
-        + (
-            '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
-            % (actions, global_flags),
-            '    (check_version &)',
-            '}',
-            '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
-        )
-    )
-
-
-# fish section
+import borgmatic.commands.arguments
+import borgmatic.commands.completion.actions
 
 
 def has_file_options(action: Action):
@@ -155,18 +91,22 @@ def fish_completion():
     Return a fish completion script for the borgmatic command. Produce this by introspecting
     borgmatic's command-line argument parsers.
     '''
-    top_level_parser, subparsers = arguments.make_parsers()
+    (
+        unused_global_parser,
+        action_parsers,
+        global_plus_action_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
 
-    all_subparsers = ' '.join(action for action in subparsers.choices.keys())
+    all_action_parsers = ' '.join(action for action in action_parsers.choices.keys())
 
     exact_option_args = tuple(
         ' '.join(action.option_strings)
-        for subparser in subparsers.choices.values()
-        for action in subparser._actions
+        for action_parser in action_parsers.choices.values()
+        for action in action_parser._actions
         if has_exact_options(action)
     ) + tuple(
         ' '.join(action.option_strings)
-        for action in top_level_parser._actions
+        for action in global_plus_action_parser._actions
         if len(action.option_strings) > 0
         if has_exact_options(action)
     )
@@ -182,7 +122,7 @@ def fish_completion():
                         set this_script (cat $this_filename 2> /dev/null)
                         set installed_script (borgmatic --fish-completion 2> /dev/null)
                         if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]
-                            echo "{upgrade_message(
+                            echo "{borgmatic.commands.completion.actions.upgrade_message(
                             'fish',
                             'borgmatic --fish-completion | sudo tee $this_filename',
                             '$this_filename',
@@ -208,29 +148,29 @@ def fish_completion():
                 return 1
             end
 
-            set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}"
+            set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}"
             set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
             '''
         )
-        + ('\n# subparser completions',)
+        + ('\n# action_parser completions',)
         + tuple(
-            f'''complete -c borgmatic -f -n "$subparser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}'''
-            for action_name, subparser in subparsers.choices.items()
+            f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}'''
+            for action_name, action_parser in action_parsers.choices.items()
         )
         + ('\n# global flags',)
         + tuple(
             # -n is checked in order, so put faster / more likely to be true checks first
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
-            for action in top_level_parser._actions
+            for action in global_plus_action_parser._actions
             # ignore the noargs action, as this is an impossible completion for fish
             if len(action.option_strings) > 0
             if 'Deprecated' not in action.help
         )
-        + ('\n# subparser flags',)
+        + ('\n# action_parser flags',)
         + tuple(
             f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
-            for action_name, subparser in subparsers.choices.items()
-            for action in subparser._actions
-            if 'Deprecated' not in action.help
+            for action_name, action_parser in action_parsers.choices.items()
+            for action in action_parser._actions
+            if 'Deprecated' not in (action.help or ())
         )
     )

borgmatic/commands/convert_config.py

@@ -1,102 +0,0 @@
-import os
-import sys
-import textwrap
-from argparse import ArgumentParser
-
-from ruamel import yaml
-
-from borgmatic.config import convert, generate, legacy, validate
-
-DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
-DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
-DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
-
-
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    parser = ArgumentParser(
-        description='''
-            Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
-            configuration file. Note that this replaces any comments from the source files.
-        '''
-    )
-    parser.add_argument(
-        '-s',
-        '--source-config',
-        dest='source_config_filename',
-        default=DEFAULT_SOURCE_CONFIG_FILENAME,
-        help=f'Source INI-style configuration filename. Default: {DEFAULT_SOURCE_CONFIG_FILENAME}',
-    )
-    parser.add_argument(
-        '-e',
-        '--source-excludes',
-        dest='source_excludes_filename',
-        default=DEFAULT_SOURCE_EXCLUDES_FILENAME
-        if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME)
-        else None,
-        help='Excludes filename',
-    )
-    parser.add_argument(
-        '-d',
-        '--destination-config',
-        dest='destination_config_filename',
-        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help=f'Destination YAML configuration filename. Default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
-    )
-
-    return parser.parse_args(arguments)
-
-
-TEXT_WRAP_CHARACTERS = 80
-
-
-def display_result(args):  # pragma: no cover
-    result_lines = textwrap.wrap(
-        f'Your borgmatic configuration has been upgraded. Please review the result in {args.destination_config_filename}.',
-        TEXT_WRAP_CHARACTERS,
-    )
-
-    excludes_phrase = (
-        f' and {args.source_excludes_filename}' if args.source_excludes_filename else ''
-    )
-    delete_lines = textwrap.wrap(
-        f'Once you are satisfied, you can safely delete {args.source_config_filename}{excludes_phrase}.',
-        TEXT_WRAP_CHARACTERS,
-    )
-
-    print('\n'.join(result_lines))
-    print()
-    print('\n'.join(delete_lines))
-
-
-def main():  # pragma: no cover
-    try:
-        args = parse_arguments(*sys.argv[1:])
-        schema = yaml.round_trip_load(open(validate.schema_filename()).read())
-        source_config = legacy.parse_configuration(
-            args.source_config_filename, legacy.CONFIG_FORMAT
-        )
-        source_config_file_mode = os.stat(args.source_config_filename).st_mode
-        source_excludes = (
-            open(args.source_excludes_filename).read().splitlines()
-            if args.source_excludes_filename
-            else []
-        )
-
-        destination_config = convert.convert_legacy_parsed_config(
-            source_config, source_excludes, schema
-        )
-
-        generate.write_configuration(
-            args.destination_config_filename,
-            generate.render_configuration(destination_config),
-            mode=source_config_file_mode,
-        )
-
-        display_result(args)
-    except (ValueError, OSError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)

borgmatic/commands/generate_config.py

@@ -1,63 +1,17 @@
+import logging
 import sys
-from argparse import ArgumentParser
 
-from borgmatic.config import generate, validate
-
-DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+import borgmatic.commands.borgmatic
 
 
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
-    parser.add_argument(
-        '-s',
-        '--source',
-        dest='source_filename',
-        help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration',
-    )
-    parser.add_argument(
-        '-d',
-        '--destination',
-        dest='destination_filename',
-        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
-    )
-    parser.add_argument(
-        '--overwrite',
-        default=False,
-        action='store_true',
-        help='Whether to overwrite any existing destination file, defaults to false',
-    )
-
-    return parser.parse_args(arguments)
-
-
-def main():  # pragma: no cover
-    try:
-        args = parse_arguments(*sys.argv[1:])
-
-        generate.generate_sample_configuration(
-            args.source_filename,
-            args.destination_filename,
-            validate.schema_filename(),
-            overwrite=args.overwrite,
+def main():
+    warning_log = logging.makeLogRecord(
+        dict(
+            levelno=logging.WARNING,
+            levelname='WARNING',
+            msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
         )
+    )
 
-        print(f'Generated a sample configuration file at {args.destination_filename}.')
-        print()
-        if args.source_filename:
-            print(f'Merged in the contents of configuration file at {args.source_filename}.')
-            print('To review the changes made, run:')
-            print()
-            print(f'    diff --unified {args.source_filename} {args.destination_filename}')
-            print()
-        print('This includes all available configuration options with example values. The few')
-        print('required options are indicated. Please edit the file to suit your needs.')
-        print()
-        print('If you ever need help: https://torsion.org/borgmatic/#issues')
-    except (ValueError, OSError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)
+    sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
+    borgmatic.commands.borgmatic.main([warning_log])

borgmatic/commands/validate_config.py

@@ -1,68 +1,17 @@
 import logging
 import sys
-from argparse import ArgumentParser
 
-import borgmatic.config.generate
-from borgmatic.config import collect, validate
-
-logger = logging.getLogger(__name__)
+import borgmatic.commands.borgmatic
 
 
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    config_paths = collect.get_default_config_paths()
-
-    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
-    parser.add_argument(
-        '-c',
-        '--config',
-        nargs='+',
-        dest='config_paths',
-        default=config_paths,
-        help=f'Configuration filenames or directories, defaults to: {config_paths}',
-    )
-    parser.add_argument(
-        '-s',
-        '--show',
-        action='store_true',
-        help='Show the validated configuration after all include merging has occurred',
+def main():
+    warning_log = logging.makeLogRecord(
+        dict(
+            levelno=logging.WARNING,
+            levelname='WARNING',
+            msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.',
+        )
     )
 
-    return parser.parse_args(arguments)
-
-
-def main():  # pragma: no cover
-    arguments = parse_arguments(*sys.argv[1:])
-
-    logging.basicConfig(level=logging.INFO, format='%(message)s')
-
-    config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths))
-    if len(config_filenames) == 0:
-        logger.critical('No files to validate found')
-        sys.exit(1)
-
-    found_issues = False
-    for config_filename in config_filenames:
-        try:
-            config, parse_logs = validate.parse_configuration(
-                config_filename, validate.schema_filename()
-            )
-        except (ValueError, OSError, validate.Validation_error) as error:
-            logging.critical(f'{config_filename}: Error parsing configuration file')
-            logging.critical(error)
-            found_issues = True
-        else:
-            for log in parse_logs:
-                logger.handle(log)
-
-            if arguments.show:
-                print('---')
-                print(borgmatic.config.generate.render_configuration(config))
-
-    if found_issues:
-        sys.exit(1)
-
-    logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")
+    sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:]
+    borgmatic.commands.borgmatic.main([warning_log])

borgmatic/config/collect.py

@@ -24,9 +24,9 @@ def get_default_config_paths(expand_home=True):
 def collect_config_filenames(config_paths):
     '''
     Given a sequence of config paths, both filenames and directories, resolve that to an iterable
-    of files. Accomplish this by listing any given directories looking for contained config files
-    (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given
-    directories are ignored.
+    of absolute files. Accomplish this by listing any given directories looking for contained config
+    files (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories
+    within the given directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
     configuration paths. However, skip a default config path if it's missing, so the user doesn't
@@ -41,7 +41,7 @@ def collect_config_filenames(config_paths):
             continue
 
         if not os.path.isdir(path) or not exists:
-            yield path
+            yield os.path.abspath(path)
             continue
 
         if not os.access(path, os.R_OK):
@@ -51,4 +51,4 @@ def collect_config_filenames(config_paths):
             full_filename = os.path.join(path, filename)
             matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
             if matching_filetype and not os.path.isdir(full_filename):
-                yield full_filename
+                yield os.path.abspath(full_filename)

borgmatic/config/convert.py

@@ -1,95 +0,0 @@
-import os
-
-from ruamel import yaml
-
-from borgmatic.config import generate
-
-
-def _convert_section(source_section_config, section_schema):
-    '''
-    Given a legacy Parsed_config instance for a single section, convert it to its corresponding
-    yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
-
-    Where integer types exist in the given section schema, convert their values to integers.
-    '''
-    destination_section_config = yaml.comments.CommentedMap(
-        [
-            (
-                option_name,
-                int(option_value)
-                if section_schema['properties'].get(option_name, {}).get('type') == 'integer'
-                else option_value,
-            )
-            for option_name, option_value in source_section_config.items()
-        ]
-    )
-
-    return destination_section_config
-
-
-def convert_legacy_parsed_config(source_config, source_excludes, schema):
-    '''
-    Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
-    patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
-    preparation for serialization to a single YAML config file.
-
-    Additionally, use the given schema as a source of helpful comments to include within the
-    returned CommentedMap.
-    '''
-    destination_config = yaml.comments.CommentedMap(
-        [
-            (section_name, _convert_section(section_config, schema['properties'][section_name]))
-            for section_name, section_config in source_config._asdict().items()
-        ]
-    )
-
-    # Split space-separated values into actual lists, make "repository" into a list, and merge in
-    # excludes.
-    location = destination_config['location']
-    location['source_directories'] = source_config.location['source_directories'].split(' ')
-    location['repositories'] = [location.pop('repository')]
-    location['exclude_patterns'] = source_excludes
-
-    if source_config.consistency.get('checks'):
-        destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
-
-    # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration_object(destination_config, schema)
-
-    for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration_object(
-            section_config, schema['properties'][section_name], indent=generate.INDENT
-        )
-
-    return destination_config
-
-
-class Legacy_configuration_not_upgraded(FileNotFoundError):
-    def __init__(self):
-        super(Legacy_configuration_not_upgraded, self).__init__(
-            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
-to YAML. This better supports validation, and has a more natural way to express
-lists of values. To upgrade your existing configuration, run:
-
-    sudo upgrade-borgmatic-config
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and excludes
-files. The new version of borgmatic will consume the YAML configuration file
-instead of the old one.'''
-        )
-
-
-def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
-    '''
-    If legacy source configuration exists but no destination upgraded configs do, raise
-    Legacy_configuration_not_upgraded.
-
-    The idea is that we want to alert the user about upgrading their config if they haven't already.
-    '''
-    destination_config_exists = any(
-        os.path.exists(filename) for filename in destination_config_filenames
-    )
-
-    if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise Legacy_configuration_not_upgraded()

borgmatic/config/generate.py

@@ -267,7 +267,7 @@ def merge_source_configuration_into_destination(destination_config, source_confi
 
 
 def generate_sample_configuration(
-    source_filename, destination_filename, schema_filename, overwrite=False
+    dry_run, source_filename, destination_filename, schema_filename, overwrite=False
 ):
     '''
     Given an optional source configuration filename, and a required destination configuration
@@ -287,6 +287,9 @@ def generate_sample_configuration(
         _schema_to_sample_configuration(schema), source_config
     )
 
+    if dry_run:
+        return
+
     write_configuration(
         destination_filename,
         _comment_out_optional_configuration(render_configuration(destination_config)),

borgmatic/config/schema.yaml

@@ -763,10 +763,21 @@ properties:
                                 Database hostname to connect to. Defaults to
                                 connecting via local Unix socket.
                             example: database.example.org
+                        restore_hostname:
+                            type: string
+                            description: |
+                                Database hostname to restore to. Defaults to
+                                the "hostname" option.
+                            example: database.example.org
                         port:
                             type: integer
                             description: Port to connect to. Defaults to 5432.
                             example: 5433
+                        restore_port:
+                            type: integer
+                            description: Port to restore to. Defaults to the
+                                "port" option.
+                            example: 5433
                         username:
                             type: string
                             description: |
@@ -775,6 +786,12 @@ properties:
                                 You probably want to specify the "postgres"
                                 superuser here when the database name is "all".
                             example: dbuser
+                        restore_username:
+                            type: string
+                            description: |
+                                Username with which to restore the database.
+                                Defaults to the "username" option.
+                            example: dbuser
                         password:
                             type: string
                             description: |
@@ -784,6 +801,24 @@ properties:
                                 without a password or you create a ~/.pgpass
                                 file.
                             example: trustsome1
+                        restore_password:
+                            type: string
+                            description: |
+                                Password with which to connect to the restore
+                                database. Defaults to the "password" option.
+                            example: trustsome1
+                        no_owner:
+                            type: boolean
+                            description: |
+                                Do not output commands to set ownership of
+                                objects to match the original database. By
+                                default, pg_dump and pg_restore issue ALTER 
+                                OWNER or SET SESSION AUTHORIZATION statements 
+                                to set ownership of created schema elements. 
+                                These statements will fail unless the initial
+                                connection to the database is made by a 
+                                superuser.
+                            example: true
                         format:
                             type: string
                             enum: ['plain', 'custom', 'directory', 'tar']
@@ -919,16 +954,33 @@ properties:
                                 Database hostname to connect to. Defaults to
                                 connecting via local Unix socket.
                             example: database.example.org
+                        restore_hostname:
+                            type: string
+                            description: |
+                                Database hostname to restore to. Defaults to
+                                the "hostname" option.
+                            example: database.example.org
                         port:
                             type: integer
                             description: Port to connect to. Defaults to 3306.
                             example: 3307
+                        restore_port:
+                            type: integer
+                            description: Port to restore to. Defaults to the
+                                "port" option.
+                            example: 5433
                         username:
                             type: string
                             description: |
                                 Username with which to connect to the database.
                                 Defaults to the username of the current user.
                             example: dbuser
+                        restore_username:
+                            type: string
+                            description: |
+                                Username with which to restore the database.
+                                Defaults to the "username" option.
+                            example: dbuser                        
                         password:
                             type: string
                             description: |
@@ -937,6 +989,12 @@ properties:
                                 configured to trust the configured username
                                 without a password.
                             example: trustsome1
+                        restore_password:
+                            type: string
+                            description: |
+                                Password with which to connect to the restore
+                                database. Defaults to the "password" option.
+                            example: trustsome1                        
                         format:
                             type: string
                             enum: ['sql']
@@ -1014,6 +1072,12 @@ properties:
                                 read_special and one_file_system (see above) to
                                 support dump and restore streaming.
                             example: /var/lib/sqlite/users.db
+                        restore_path:
+                            type: string
+                            description: |
+                                Path to the SQLite database file to restore to.
+                                Defaults to the "path" option.
+                            example: /var/lib/sqlite/users.db
             mongodb_databases:
                 type: array
                 items:
@@ -1036,22 +1100,45 @@ properties:
                                 Database hostname to connect to. Defaults to
                                 connecting to localhost.
                             example: database.example.org
+                        restore_hostname:
+                            type: string
+                            description: |
+                                Database hostname to restore to. Defaults to
+                                the "hostname" option.
+                            example: database.example.org
                         port:
                             type: integer
                             description: Port to connect to. Defaults to 27017.
                             example: 27018
+                        restore_port:
+                            type: integer
+                            description: Port to restore to. Defaults to the
+                                "port" option.
+                            example: 5433                        
                         username:
                             type: string
                             description: |
                                 Username with which to connect to the database.
                                 Skip it if no authentication is needed.
                             example: dbuser
+                        restore_username:
+                            type: string
+                            description: |
+                                Username with which to restore the database.
+                                Defaults to the "username" option.
+                            example: dbuser                        
                         password:
                             type: string
                             description: |
                                 Password with which to connect to the database.
                                 Skip it if no authentication is needed.
                             example: trustsome1
+                        restore_password:
+                            type: string
+                            description: |
+                                Password with which to connect to the restore
+                                database. Defaults to the "password" option.
+                            example: trustsome1
                         authentication_database:
                             type: string
                             description: |

borgmatic/hooks/mongodb.py

@@ -102,7 +102,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given MongoDB database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -122,7 +124,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(location_config), database['name'], database.get('hostname')
     )
-    restore_command = build_restore_command(extract_process, database, dump_filename)
+    restore_command = build_restore_command(
+        extract_process, database, dump_filename, connection_params
+    )
 
     logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
     if dry_run:
@@ -138,10 +142,21 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     )
 
 
-def build_restore_command(extract_process, database, dump_filename):
+def build_restore_command(extract_process, database, dump_filename, connection_params):
     '''
     Return the mongorestore command from a single database configuration.
     '''
+    hostname = connection_params['hostname'] or database.get(
+        'restore_hostname', database.get('hostname')
+    )
+    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
+    username = connection_params['username'] or database.get(
+        'restore_username', database.get('username')
+    )
+    password = connection_params['password'] or database.get(
+        'restore_password', database.get('password')
+    )
+
     command = ['mongorestore']
     if extract_process:
         command.append('--archive')
@@ -149,14 +164,14 @@ def build_restore_command(extract_process, database, dump_filename):
         command.extend(('--dir', dump_filename))
     if database['name'] != 'all':
         command.extend(('--drop', '--db', database['name']))
-    if 'hostname' in database:
-        command.extend(('--host', database['hostname']))
-    if 'port' in database:
-        command.extend(('--port', str(database['port'])))
-    if 'username' in database:
-        command.extend(('--username', database['username']))
-    if 'password' in database:
-        command.extend(('--password', database['password']))
+    if hostname:
+        command.extend(('--host', hostname))
+    if port:
+        command.extend(('--port', str(port)))
+    if username:
+        command.extend(('--username', username))
+    if password:
+        command.extend(('--password', password))
     if 'authentication_database' in database:
         command.extend(('--authenticationDatabase', database['authentication_database']))
     if 'restore_options' in database:

borgmatic/hooks/mysql.py

@@ -185,7 +185,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -199,15 +201,27 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         raise ValueError('The database configuration value is invalid')
 
     database = database_config[0]
+
+    hostname = connection_params['hostname'] or database.get(
+        'restore_hostname', database.get('hostname')
+    )
+    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
+    username = connection_params['username'] or database.get(
+        'restore_username', database.get('username')
+    )
+    password = connection_params['password'] or database.get(
+        'restore_password', database.get('password')
+    )
+
     restore_command = (
         ('mysql', '--batch')
         + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
-        + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (('--host', hostname) if hostname else ())
+        + (('--port', str(port)) if port else ())
+        + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--user', username) if username else ())
     )
-    extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+    extra_environment = {'MYSQL_PWD': password} if password else None
 
     logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/postgresql.py

@@ -23,13 +23,23 @@ def make_dump_path(location_config):  # pragma: no cover
     )
 
 
-def make_extra_environment(database):
+def make_extra_environment(database, restore_connection_params=None):
     '''
     Make the extra_environment dict from the given database configuration.
+    If restore connection params are given, this is for a restore operation.
     '''
     extra = dict()
-    if 'password' in database:
-        extra['PGPASSWORD'] = database['password']
+
+    try:
+        if restore_connection_params:
+            extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
+                'restore_password', database['password']
+            )
+        else:
+            extra['PGPASSWORD'] = database['password']
+    except (AttributeError, KeyError):
+        pass
+
     extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
     if 'ssl_cert' in database:
         extra['PGSSLCERT'] = database['ssl_cert']
@@ -135,6 +145,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
                 + (('--host', database['hostname']) if 'hostname' in database else ())
                 + (('--port', str(database['port'])) if 'port' in database else ())
                 + (('--username', database['username']) if 'username' in database else ())
+                + (('--no-owner',) if database.get('no_owner', False) else ())
                 + (('--format', dump_format) if dump_format else ())
                 + (('--file', dump_filename) if dump_format == 'directory' else ())
                 + (tuple(database['options'].split(' ')) if 'options' in database else ())
@@ -192,7 +203,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given PostgreSQL database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -202,6 +215,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
 
     If the extract process is None, then restore the dump from the filesystem rather than from an
     extract stream.
+
+    Use the given connection parameters to connect to the database. The connection parameters are
+    hostname, port, username, and password.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
@@ -209,6 +225,15 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         raise ValueError('The database configuration value is invalid')
 
     database = database_config[0]
+
+    hostname = connection_params['hostname'] or database.get(
+        'restore_hostname', database.get('hostname')
+    )
+    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
+    username = connection_params['username'] or database.get(
+        'restore_username', database.get('username')
+    )
+
     all_databases = bool(database['name'] == 'all')
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(location_config), database['name'], database.get('hostname')
@@ -217,9 +242,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     analyze_command = (
         tuple(psql_command)
         + ('--no-password', '--no-psqlrc', '--quiet')
-        + (('--host', database['hostname']) if 'hostname' in database else ())
-        + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--username', database['username']) if 'username' in database else ())
+        + (('--host', hostname) if hostname else ())
+        + (('--port', port) if port else ())
+        + (('--username', username) if username else ())
         + (('--dbname', database['name']) if not all_databases else ())
         + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
         + ('--command', 'ANALYZE')
@@ -231,9 +256,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         + ('--no-password',)
         + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
         + (('--dbname', database['name']) if not all_databases else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
-        + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--username', database['username']) if 'username' in database else ())
+        + (('--host', hostname) if hostname else ())
+        + (('--port', port) if port else ())
+        + (('--username', username) if username else ())
+        + (('--no-owner',) if database.get('no_owner', False) else ())
         + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
         + (() if extract_process else (dump_filename,))
         + tuple(
@@ -243,7 +269,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         )
     )
 
-    extra_environment = make_extra_environment(database)
+    extra_environment = make_extra_environment(
+        database, restore_connection_params=connection_params
+    )
 
     logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/sqlite.py

@@ -85,7 +85,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name)
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given SQLite3 database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -98,7 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     if len(database_config) != 1:
         raise ValueError('The database configuration value is invalid')
 
-    database_path = database_config[0]['path']
+    database_path = connection_params['restore_path'] or database_config[0].get(
+        'restore_path', database_config[0].get('path')
+    )
 
     logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
     if dry_run:

docs/Dockerfile

@@ -4,9 +4,9 @@ COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in rcreate transfer create prune compact check extract export-tar mount umount restore rlist list rinfo info break-lock borg; do \
+    && for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" "config validate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
-           && borgmatic "$action" --help >> /command-line.txt; done
+           && borgmatic $action --help >> /command-line.txt; done
 
 FROM docker.io/node:19.5.0-alpine as html
 

docs/how-to/backup-your-databases.md

@@ -348,17 +348,19 @@ borgmatic's own configuration file. So include your configuration file in
 backups to avoid getting caught without a way to restore a database.
 3. borgmatic does not currently support backing up or restoring multiple
 databases that share the exact same name on different hosts.
-4. Because database hooks implicitly enable the `read_special` configuration
-setting to support dump and restore streaming, you'll need to ensure that any
-special files are excluded from backups (named pipes, block devices,
-character devices, and sockets) to prevent hanging. Try a command like
-`find /your/source/path -type b -or -type c -or -type p -or -type s` to find
-such files. Common directories to exclude are `/dev` and `/run`, but that may
-not be exhaustive. <span class="minilink minilink-addedin">New in version
+4. Because database hooks implicitly enable the `read_special` configuration,
+any special files are excluded from backups (named pipes, block devices,
+character devices, and sockets) to prevent hanging. Try a command like `find
+/your/source/path -type b -or -type c -or -type p -or -type s` to find such
+files. Common directories to exclude are `/dev` and `/run`, but that may not
+be exhaustive. <span class="minilink minilink-addedin">New in version
 1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
-special files that may cause Borg to hang, so you no longer need to manually
-exclude them. (This includes symlinks with special files as a destination.) You
-can override/prevent this behavior by explicitly setting `read_special` to true.
+special files (and symlinks to special files) that may cause Borg to hang, so
+generally you no longer need to manually exclude them. There are potential
+edge cases though in which applications on your system create new special files
+*after* borgmatic constructs its exclude list, resulting in Borg hangs. If that
+occurs, you can resort to the manual excludes described above. And to opt out
+of the auto-exclude feature entirely, explicitly set `read_special` to true.
 
 
 ### Manual restoration
@@ -408,6 +410,12 @@ authenticated. For instance, with PostgreSQL, check your
 [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html)
 file for that configuration.
 
+Additionally, MySQL/MariaDB may be picking up some of your credentials from a
+defaults file like `~/.my.cnf`. If that's the case, then it's possible
+MySQL/MariaDB ends up using, say, a username from borgmatic's configuration
+and a password from `~/.my.cnf`. This may result in authentication errors if
+this combination of credentials is not what you intend.
+
 
 ### MySQL table lock errors
 

docs/how-to/make-per-application-backups.md

@@ -20,18 +20,22 @@ instance, for applications:
 
 ```bash
 sudo mkdir /etc/borgmatic.d
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/app1.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/app2.yaml
 ```
 
 Or, for repositories:
 
 ```bash
 sudo mkdir /etc/borgmatic.d
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo1.yaml
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo2.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/repo1.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/repo2.yaml
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span> The
+command to generate configuration files was `generate-borgmatic-config`
+instead of `borgmatic config generate`.
+
 When you set up multiple configuration files like this, borgmatic will run
 each one in turn from a single borgmatic invocation. This includes, by
 default, the traditional `/etc/borgmatic/config.yaml` as well.
@@ -395,9 +399,16 @@ includes.
 
 ## Debugging includes
 
-<span class="minilink minilink-addedin">New in version 1.7.12</span> If you'd
+<span class="minilink minilink-addedin">New in version 1.7.15</span> If you'd
 like to see what the loaded configuration looks like after includes get merged
-in, run `validate-borgmatic-config` on your configuration file:
+in, run the `validate` action on your configuration file:
+
+```bash
+sudo borgmatic config validate --show
+```
+
+<span class="minilink minilink-addedin">In version 1.7.12 through
+1.7.14</span> Use this command instead:
 
 ```bash
 sudo validate-borgmatic-config --show

docs/how-to/set-up-backups.md

@@ -89,7 +89,7 @@ installing borgmatic:
  * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
  * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/)
  * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic)
- * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
+ * [Arch Linux](https://archlinux.org/packages/extra/any/borgmatic/)
  * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
  * [OpenBSD](https://openports.pl/path/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
@@ -120,16 +120,24 @@ offerings, but do not currently fund borgmatic development or hosting.
 
 After you install borgmatic, generate a sample configuration file:
 
+```bash
+sudo borgmatic config generate
+```
+
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span>
+Generate a configuration file with this command instead:
+
 ```bash
 sudo generate-borgmatic-config
 ```
 
-If that command is not found, then it may be installed in a location that's
-not in your system `PATH` (see above). Try looking in `~/.local/bin/`.
+If neither command is found, then borgmatic may be installed in a location
+that's not in your system `PATH` (see above). Try looking in `~/.local/bin/`.
 
-This generates a sample configuration file at `/etc/borgmatic/config.yaml` by
-default. If you'd like to use another path, use the `--destination` flag, for
-instance: `--destination ~/.config/borgmatic/config.yaml`.
+The command generates a sample configuration file at
+`/etc/borgmatic/config.yaml` by default. If you'd like to use another path,
+use the `--destination` flag, for instance: `--destination
+~/.config/borgmatic/config.yaml`.
 
 You should edit the configuration file to suit your needs, as the generated
 values are only representative. All options are optional except where
@@ -177,6 +185,13 @@ redundant](https://torsion.org/borgmatic/docs/how-to/make-backups-redundant/).
 If you'd like to validate that your borgmatic configuration is valid, the
 following command is available for that:
 
+```bash
+sudo borgmatic config validate
+```
+
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span>
+Validate a configuration file with this command instead:
+
 ```bash
 sudo validate-borgmatic-config
 ```

docs/how-to/upgrade.md

@@ -29,29 +29,33 @@ configuration options. This is completely optional. If you prefer, you can add
 new configuration options manually.
 
 If you do want to upgrade your configuration file to include new options, use
-the `generate-borgmatic-config` script with its optional `--source` flag that
+the `borgmatic config generate` action with its optional `--source` flag that
 takes the path to your original configuration file. If provided with this
-path, `generate-borgmatic-config` merges your original configuration into the
+path, `borgmatic config generate` merges your original configuration into the
 generated configuration file, so you get all the newest options and comments.
 
 Here's an example:
 
 ```bash
-generate-borgmatic-config --source config.yaml --destination config-new.yaml
+borgmatic config generate --source config.yaml --destination config-new.yaml
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span> The
+command to generate configuration files was `generate-borgmatic-config`
+instead of `borgmatic config generate`.
+
 New options start as commented out, so you can edit the file and decide
 whether you want to use each one.
 
 There are a few caveats to this process. First, when generating the new
-configuration file, `generate-borgmatic-config` replaces any comments you've
+configuration file, `borgmatic config generate` replaces any comments you've
 written in your original configuration file with the newest generated
 comments. Second, the script adds back any options you had originally deleted,
 although it does so with the options commented out. And finally, any YAML
 includes you've used in the source configuration get flattened out into a
 single generated file.
 
-As a safety measure, `generate-borgmatic-config` refuses to modify
+As a safety measure, `borgmatic config generate` refuses to modify
 configuration files in-place. So it's up to you to review the generated file
 and, if desired, replace your original configuration file with it.
 
@@ -61,21 +65,22 @@ and, if desired, replace your original configuration file with it.
 borgmatic changed its configuration file format in version 1.1.0 from
 INI-style to YAML. This better supports validation, and has a more natural way
 to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic.
+upgrade to the last version of borgmatic to support converting configuration:
+borgmatic 1.7.14.
 
 As of version 1.1.0, borgmatic no longer supports Python 2. If you were
 already running borgmatic with Python 3, then you can upgrade borgmatic
 in-place:
 
 ```bash
-sudo pip3 install --user --upgrade borgmatic
+sudo pip3 install --user --upgrade borgmatic==1.7.14
 ```
 
 But if you were running borgmatic with Python 2, uninstall and reinstall instead:
 
 ```bash
 sudo pip uninstall borgmatic
-sudo pip3 install --user borgmatic
+sudo pip3 install --user borgmatic==1.7.14
 ```
 
 The pip binary names for different versions of Python can differ, so the above
@@ -93,29 +98,12 @@ That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
 excludes files. The new version of borgmatic will consume the YAML
 configuration file instead of the old one.
 
-
-### Upgrading from atticmatic
-
-You can ignore this section if you're not an atticmatic user (the former name
-of borgmatic).
-
-borgmatic only supports Borg now and no longer supports Attic. So if you're
-an Attic user, consider switching to Borg. See the [Borg upgrade
-command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
-for more information. Then, follow the instructions above about setting up
-your borgmatic configuration files.
-
-If you were already using Borg with atticmatic, then you can upgrade
-from atticmatic to borgmatic by running the following commands:
+Now you can upgrade to a newer version of borgmatic:
 
 ```bash
-sudo pip3 uninstall atticmatic
 sudo pip3 install --user borgmatic
 ```
 
-That's it! borgmatic will continue using your /etc/borgmatic configuration
-files.
-
 
 ## Upgrading Borg
 

docs/reference/command-line.md

@@ -7,11 +7,18 @@ eleventyNavigation:
 ---
 ## borgmatic options
 
-Here are all of the available borgmatic command-line options, including the
-separate options for each action sub-command. Note that most of the
-flags listed here do not have equivalents in borgmatic's [configuration
+Here are all of the available borgmatic command-line flags for the [most
+recent version of
+borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/releases),
+including the separate flags for each action (sub-command). Most of the flags
+listed here do not have equivalents in borgmatic's [configuration
 file](https://torsion.org/borgmatic/docs/reference/configuration/).
 
+If you're using an older version of borgmatic, some of these flags may not be
+present in that version, and you should instead use `borgmatic --help` or
+`borgmatic [action name] --help` (where `[action name]` is the name of an
+action like `list`, `create`, etc.).
+
 ```
 {% include borgmatic/command-line.txt %}
 ```

docs/reference/configuration.md

@@ -7,11 +7,19 @@ eleventyNavigation:
 ---
 ## Configuration file
 
-Here is a full sample borgmatic configuration file including all available options: 
+Below is a sample borgmatic configuration file including all available options
+for the [most recent version of
+borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/releases).
+This file is also [available for
+download](https://torsion.org/borgmatic/docs/reference/config.yaml).
+
+If you're using an older version of borgmatic, some of these options may not
+work, and you should instead [generate a sample configuration file specific to
+your borgmatic
+version](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#configuration).
 
 ```yaml
 {% include borgmatic/config.yaml %}
 ```
 
 Note that you can also [download this configuration
-file](https://torsion.org/borgmatic/docs/reference/config.yaml) for use locally.

sample/systemd/borgmatic-user.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+ConditionACPower=true
+
+[Service]
+Type=oneshot
+Restart=no
+# Prevent rate limiting of borgmatic log events. If you are using an older version of systemd that
+# doesn't support this (pre-240 or so), you may have to remove this option.
+LogRateLimitIntervalSec=0
+# Delay start to prevent backups running during boot.
+ExecStartPre=sleep 1m
+ExecStart=/root/.local/bin/borgmatic --verbosity -2 --syslog-verbosity 1

sample/systemd/borgmatic-user.timer

@@ -0,0 +1 @@
+borgmatic.timer

scripts/release

@@ -35,10 +35,8 @@ git push github $version
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
-gpg --detach-sign --armor dist/borgmatic-*.tar.gz
-gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl
-twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
-twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none-any.whl.asc
+twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz
+twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl
 
 # Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
@@ -50,5 +48,5 @@ curl --silent --request POST \
     --header "Content-Type: application/json" \
     --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}"
 
-github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" \
+github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" --target_commit="main" \
     --name="borgmatic $version" --body="$release_changelog"

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.7.14'
+VERSION = '1.7.15'
 
 
 setup(
@@ -23,7 +23,6 @@ setup(
     entry_points={
         'console_scripts': [
             'borgmatic = borgmatic.commands.borgmatic:main',
-            'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
             'generate-borgmatic-config = borgmatic.commands.generate_config:main',
             'validate-borgmatic-config = borgmatic.commands.validate_config:main',
         ]

tests/end-to-end/docker-compose.yaml

@@ -5,16 +5,35 @@ services:
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
+  postgresql2:
+    image: docker.io/postgres:13.1-alpine
+    environment:
+      POSTGRES_PASSWORD: test2
+      POSTGRES_DB: test
+      POSTGRES_USER: postgres2
+    command: docker-entrypoint.sh -p 5433
   mysql:
     image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
+  mysql2:
+    image: docker.io/mariadb:10.5
+    environment:
+      MYSQL_ROOT_PASSWORD: test2
+      MYSQL_DATABASE: test
+    command: docker-entrypoint.sh --port=3307
   mongodb:
     image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
+  mongodb2:
+    image: docker.io/mongo:5.0.5
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root2
+      MONGO_INITDB_ROOT_PASSWORD: test2
+    command: docker-entrypoint.sh --port=27018
   tests:
     image: docker.io/alpine:3.13
     environment:
@@ -30,5 +49,8 @@ services:
     command: --end-to-end-only
     depends_on:
       - postgresql
+      - postgresql2
       - mysql
+      - mysql2
       - mongodb
+      - mongodb2

tests/end-to-end/test_borgmatic.py

@@ -12,7 +12,7 @@ def generate_configuration(config_path, repository_path):
     to work for testing (including injecting the given repository path and tacking on an encryption
     passphrase).
     '''
-    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
     config = (
         open(config_path)
         .read()
@@ -44,7 +44,7 @@ def test_borgmatic_command():
         generate_configuration(config_path, repository_path)
 
         subprocess.check_call(
-            f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ')
+            f'borgmatic -v 2 --config {config_path} rcreate --encryption repokey'.split(' ')
         )
 
         # Run borgmatic to generate a backup archive, and then list it to make sure it exists.

tests/end-to-end/test_database.py

@@ -82,6 +82,108 @@ hooks:
         config_file.write(config)
 
 
+def write_custom_restore_configuration(
+    source_directory,
+    config_path,
+    repository_path,
+    borgmatic_source_directory,
+    postgresql_dump_format='custom',
+    mongodb_dump_format='archive',
+):
+    '''
+    Write out borgmatic configuration into a file at the config path. Set the options so as to work
+    for testing with custom restore options. This includes a custom restore_hostname, restore_port,
+    restore_username, restore_password and restore_path.
+    '''
+    config = f'''
+location:
+    source_directories:
+        - {source_directory}
+    repositories:
+        - {repository_path}
+    borgmatic_source_directory: {borgmatic_source_directory}
+
+storage:
+    encryption_passphrase: "test"
+
+hooks:
+    postgresql_databases:
+        - name: test
+          hostname: postgresql
+          username: postgres
+          password: test
+          format: {postgresql_dump_format}
+          restore_hostname: postgresql2
+          restore_port: 5433
+          restore_username: postgres2
+          restore_password: test2
+    mysql_databases:
+        - name: test
+          hostname: mysql
+          username: root
+          password: test
+          restore_hostname: mysql2
+          restore_port: 3307
+          restore_username: root
+          restore_password: test2
+    mongodb_databases:
+        - name: test
+          hostname: mongodb
+          username: root
+          password: test
+          authentication_database: admin
+          format: {mongodb_dump_format}
+          restore_hostname: mongodb2
+          restore_port: 27018
+          restore_username: root2
+          restore_password: test2
+    sqlite_databases:
+        - name: sqlite_test
+          path: /tmp/sqlite_test.db
+          restore_path: /tmp/sqlite_test2.db
+'''
+
+    with open(config_path, 'w') as config_file:
+        config_file.write(config)
+
+
+def write_simple_custom_restore_configuration(
+    source_directory,
+    config_path,
+    repository_path,
+    borgmatic_source_directory,
+    postgresql_dump_format='custom',
+):
+    '''
+    Write out borgmatic configuration into a file at the config path. Set the options so as to work
+    for testing with custom restore options, but this time using CLI arguments. This includes a
+    custom restore_hostname, restore_port, restore_username and restore_password as we only test
+    these options for PostgreSQL.
+    '''
+    config = f'''
+location:
+    source_directories:
+        - {source_directory}
+    repositories:
+        - {repository_path}
+    borgmatic_source_directory: {borgmatic_source_directory}
+
+storage:
+    encryption_passphrase: "test"
+
+hooks:
+    postgresql_databases:
+        - name: test
+          hostname: postgresql
+          username: postgres
+          password: test
+          format: {postgresql_dump_format}
+'''
+
+    with open(config_path, 'w') as config_file:
+        config_file.write(config)
+
+
 def test_database_dump_and_restore():
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()
@@ -100,7 +202,104 @@ def test_database_dump_and_restore():
         )
 
         subprocess.check_call(
-            ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
+        )
+
+        # Run borgmatic to generate a backup archive including a database dump.
+        subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
+
+        # Get the created archive name.
+        output = subprocess.check_output(
+            ['borgmatic', '--config', config_path, 'list', '--json']
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
+
+        # Restore the database from the archive.
+        subprocess.check_call(
+            ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_and_restore_with_restore_cli_arguments():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_simple_custom_restore_configuration(
+            temporary_directory, config_path, repository_path, borgmatic_source_directory
+        )
+
+        subprocess.check_call(
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
+        )
+
+        # Run borgmatic to generate a backup archive including a database dump.
+        subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
+
+        # Get the created archive name.
+        output = subprocess.check_output(
+            ['borgmatic', '--config', config_path, 'list', '--json']
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
+
+        # Restore the database from the archive.
+        subprocess.check_call(
+            [
+                'borgmatic',
+                '-v',
+                '2',
+                '--config',
+                config_path,
+                'restore',
+                '--archive',
+                archive_name,
+                '--hostname',
+                'postgresql2',
+                '--port',
+                '5433',
+                '--username',
+                'postgres2',
+                '--password',
+                'test2',
+            ]
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_and_restore_with_restore_configuration_options():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_custom_restore_configuration(
+            temporary_directory, config_path, repository_path, borgmatic_source_directory
+        )
+
+        subprocess.check_call(
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
         )
 
         # Run borgmatic to generate a backup archive including a database dump.
@@ -145,7 +344,7 @@ def test_database_dump_and_restore_with_directory_format():
         )
 
         subprocess.check_call(
-            ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
         )
 
         # Run borgmatic to generate a backup archive including a database dump.
@@ -175,7 +374,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit():
         )
 
         subprocess.check_call(
-            ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
         )
 
         # Run borgmatic with a config override such that the database dump fails.

tests/end-to-end/test_dev_parity_with_build_server.py

@@ -0,0 +1,38 @@
+import ruamel.yaml
+
+
+def test_dev_docker_compose_has_same_services_as_build_server_configuration():
+    '''
+    The end-to-end test configuration for local development and the build server's test
+    configuration use two different mechanisms for configuring and spinning up "services"—the
+    database containers upon which the end-to-end tests are reliant. The dev configuration uses
+    Docker Compose, while the Drone build server configuration uses its own similar-but-different
+    configuration file format.
+
+    Therefore, to ensure dev-build parity, these tests assert that the services are the same across
+    the dev and build configurations. This includes service name, container image, environment
+    variables, and commands.
+
+    This test only compares services and does not assert anything else about the respective testing
+    environments.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    dev_services = {
+        name: service
+        for name, service in yaml.load(open('tests/end-to-end/docker-compose.yaml').read())[
+            'services'
+        ].items()
+        if name != 'tests'
+    }
+    build_server_services = tuple(yaml.load_all(open('.drone.yml').read()))[0]['services']
+
+    assert len(dev_services) == len(build_server_services)
+
+    for build_service in build_server_services:
+        dev_service = dev_services[build_service['name']]
+        assert dev_service['image'] == build_service['image']
+        assert dev_service['environment'] == build_service['environment']
+
+        if 'command' in dev_service or 'commands' in build_service:
+            assert len(build_service['commands']) <= 1
+            assert dev_service['command'] == build_service['commands'][0]

tests/end-to-end/test_generate_config.py

@@ -8,9 +8,9 @@ def test_generate_borgmatic_config_with_merging_succeeds():
         config_path = os.path.join(temporary_directory, 'test.yaml')
         new_config_path = os.path.join(temporary_directory, 'new.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         subprocess.check_call(
-            f'generate-borgmatic-config --source {config_path} --destination {new_config_path}'.split(
+            f'borgmatic config generate --source {config_path} --destination {new_config_path}'.split(
                 ' '
             )
         )

tests/end-to-end/test_override.py

@@ -10,7 +10,7 @@ def generate_configuration(config_path, repository_path):
     to work for testing (including injecting the given repository path and tacking on an encryption
     passphrase).
     '''
-    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
     config = (
         open(config_path)
         .read()
@@ -39,7 +39,7 @@ def test_override_get_normalized():
         generate_configuration(config_path, repository_path)
 
         subprocess.check_call(
-            f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ')
+            f'borgmatic -v 2 --config {config_path} rcreate --encryption repokey'.split(' ')
         )
 
         # Run borgmatic with an override structured for an outdated config file format. If

tests/end-to-end/test_validate_config.py

@@ -8,7 +8,7 @@ def test_validate_config_command_with_valid_configuration_succeeds():
     with tempfile.TemporaryDirectory() as temporary_directory:
         config_path = os.path.join(temporary_directory, 'test.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' '))
 
         assert exit_code == 0
@@ -18,7 +18,7 @@ def test_validate_config_command_with_invalid_configuration_fails():
     with tempfile.TemporaryDirectory() as temporary_directory:
         config_path = os.path.join(temporary_directory, 'test.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"')
         config_file = open(config_path, 'w')
         config_file.write(config)
@@ -33,7 +33,7 @@ def test_validate_config_command_with_show_flag_displays_configuration():
     with tempfile.TemporaryDirectory() as temporary_directory:
         config_path = os.path.join(temporary_directory, 'test.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         output = subprocess.check_output(
             f'validate-borgmatic-config --config {config_path} --show'.split(' ')
         ).decode(sys.stdout.encoding)

tests/integration/actions/config/test_validate.py

@@ -0,0 +1,37 @@
+import argparse
+
+from flexmock import flexmock
+
+import borgmatic.logger
+from borgmatic.actions.config import validate as module
+
+
+def test_run_validate_with_show_renders_configurations():
+    log_lines = []
+    borgmatic.logger.add_custom_log_levels()
+
+    def fake_logger_answer(message):
+        log_lines.append(message)
+
+    flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
+
+    module.run_validate(argparse.Namespace(show=True), {'test.yaml': {'foo': {'bar': 'baz'}}})
+
+    assert log_lines == ['''foo:\n    bar: baz\n''']
+
+
+def test_run_validate_with_show_and_multiple_configs_renders_each():
+    log_lines = []
+    borgmatic.logger.add_custom_log_levels()
+
+    def fake_logger_answer(message):
+        log_lines.append(message)
+
+    flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
+
+    module.run_validate(
+        argparse.Namespace(show=True),
+        {'test.yaml': {'foo': {'bar': 'baz'}}, 'other.yaml': {'quux': 'value'}},
+    )
+
+    assert log_lines == ['---', 'foo:\n    bar: baz\n', '---', 'quux: value\n']

tests/integration/commands/completion/test_actions.py

@@ -0,0 +1,28 @@
+import borgmatic.commands.arguments
+from borgmatic.commands.completion import actions as module
+
+
+def test_available_actions_uses_only_subactions_for_action_with_subactions():
+    (
+        unused_global_parser,
+        action_parsers,
+        unused_combined_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
+
+    actions = module.available_actions(action_parsers, 'config')
+
+    assert 'bootstrap' in actions
+    assert 'list' not in actions
+
+
+def test_available_actions_omits_subactions_for_action_without_subactions():
+    (
+        unused_global_parser,
+        action_parsers,
+        unused_combined_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
+
+    actions = module.available_actions(action_parsers, 'list')
+
+    assert 'bootstrap' not in actions
+    assert 'config' in actions

tests/integration/commands/completion/test_bash.py

@@ -0,0 +1,5 @@
+from borgmatic.commands.completion import bash as module
+
+
+def test_bash_completion_does_not_raise():
+    assert module.bash_completion()

tests/integration/commands/completion/test_fish.py

@@ -0,0 +1,5 @@
+from borgmatic.commands.completion import fish as module
+
+
+def test_fish_completion_does_not_raise():
+    assert module.fish_completion()

tests/integration/commands/test_arguments.py

@@ -30,6 +30,28 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list():
     assert global_arguments.log_file_verbosity == 0
 
 
+def test_parse_arguments_with_action_after_config_path_omits_action():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--config', 'myconfig', 'list', '--json')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == ['myconfig']
+    assert 'list' in arguments
+    assert arguments['list'].json
+
+
+def test_parse_arguments_with_action_after_config_path_omits_aliased_action():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == ['myconfig']
+    assert 'rcreate' in arguments
+    assert arguments['rcreate'].encryption_mode == 'repokey'
+
+
 def test_parse_arguments_with_verbosity_overrides_default():
     config_paths = ['default']
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
@@ -194,10 +216,10 @@ def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
     assert 'check' in arguments
 
 
-def test_parse_arguments_with_invalid_arguments_exits():
+def test_parse_arguments_disallows_invalid_argument():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--posix-me-harder')
 
 
@@ -211,7 +233,7 @@ def test_parse_arguments_disallows_deprecated_excludes_option():
 def test_parse_arguments_disallows_encryption_mode_without_init():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
 
 
@@ -231,14 +253,14 @@ def test_parse_arguments_requires_encryption_mode_with_init():
 def test_parse_arguments_disallows_append_only_without_init():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--config', 'myconfig', '--append-only')
 
 
 def test_parse_arguments_disallows_storage_quota_without_init():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
 
 
@@ -287,17 +309,24 @@ def test_parse_arguments_allows_repository_with_list():
 def test_parse_arguments_disallows_archive_unless_action_consumes_it():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--config', 'myconfig', '--archive', 'test')
 
 
 def test_parse_arguments_disallows_paths_unless_action_consumes_it():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--config', 'myconfig', '--path', 'test')
 
 
+def test_parse_arguments_disallows_other_actions_with_config_bootstrap():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list')
+
+
 def test_parse_arguments_allows_archive_with_extract():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
@@ -373,7 +402,7 @@ def test_parse_arguments_allows_progress_and_extract():
 def test_parse_arguments_disallows_progress_without_create():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--progress', 'list')
 
 
@@ -392,7 +421,7 @@ def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
 def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    with pytest.raises(SystemExit):
+    with pytest.raises(ValueError):
         module.parse_arguments('--stats', 'list')
 
 
@@ -523,3 +552,76 @@ def test_parse_arguments_extract_with_check_only_extract_does_not_raise():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
     module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract')
+
+
+def test_parse_arguments_bootstrap_without_config_errors():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('bootstrap')
+
+
+def test_parse_arguments_config_with_no_subaction_errors():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('config')
+
+
+def test_parse_arguments_config_with_help_shows_config_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('config', '--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'global arguments:' not in captured.out
+    assert 'config arguments:' in captured.out
+    assert 'config sub-actions:' in captured.out
+
+
+def test_parse_arguments_config_with_subaction_but_missing_flags_errors():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('config', 'bootstrap')
+
+    assert exit.value.code == 2
+
+
+def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('config', 'bootstrap', '--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'config bootstrap arguments:' in captured.out
+
+
+def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg')
+
+
+def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg')
+
+
+def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1')
+
+
+def test_parse_arguments_config_with_subaction_and_explicit_config_file_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments(
+        'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml'
+    )

tests/integration/commands/test_completion.py

@@ -1,9 +0,0 @@
-from borgmatic.commands import completion as module
-
-
-def test_bash_completion_does_not_raise():
-    assert module.bash_completion()
-
-
-def test_fish_completion_does_not_raise():
-    assert module.fish_completion()

tests/integration/commands/test_convert_config.py

@@ -1,50 +0,0 @@
-import os
-
-import pytest
-from flexmock import flexmock
-
-from borgmatic.commands import convert_config as module
-
-
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
-    parser = module.parse_arguments()
-
-    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
-    assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME
-    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
-
-
-def test_parse_arguments_with_filename_arguments_overrides_defaults():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
-    parser = module.parse_arguments(
-        '--source-config',
-        'config',
-        '--source-excludes',
-        'excludes',
-        '--destination-config',
-        'config.yaml',
-    )
-
-    assert parser.source_config_filename == 'config'
-    assert parser.source_excludes_filename == 'excludes'
-    assert parser.destination_config_filename == 'config.yaml'
-
-
-def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
-    flexmock(os.path).should_receive('exists').and_return(False)
-
-    parser = module.parse_arguments()
-
-    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
-    assert parser.source_excludes_filename is None
-    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
-
-
-def test_parse_arguments_with_invalid_arguments_exits():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
-    with pytest.raises(SystemExit):
-        module.parse_arguments('--posix-me-harder')

tests/integration/commands/test_generate_config.py

@@ -1,25 +1,9 @@
+from flexmock import flexmock
+
 from borgmatic.commands import generate_config as module
 
 
-def test_parse_arguments_with_no_arguments_uses_default_destination():
-    parser = module.parse_arguments()
+def test_main_does_not_raise():
+    flexmock(module.borgmatic.commands.borgmatic).should_receive('main')
 
-    assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
-
-
-def test_parse_arguments_with_destination_argument_overrides_default():
-    parser = module.parse_arguments('--destination', 'config.yaml')
-
-    assert parser.destination_filename == 'config.yaml'
-
-
-def test_parse_arguments_parses_source():
-    parser = module.parse_arguments('--source', 'source.yaml', '--destination', 'config.yaml')
-
-    assert parser.source_filename == 'source.yaml'
-
-
-def test_parse_arguments_parses_overwrite():
-    parser = module.parse_arguments('--destination', 'config.yaml', '--overwrite')
-
-    assert parser.overwrite
+    module.main()

tests/integration/commands/test_validate_config.py

@@ -3,27 +3,7 @@ from flexmock import flexmock
 from borgmatic.commands import validate_config as module
 
 
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+def test_main_does_not_raise():
+    flexmock(module.borgmatic.commands.borgmatic).should_receive('main')
 
-    parser = module.parse_arguments()
-
-    assert parser.config_paths == config_paths
-
-
-def test_parse_arguments_with_multiple_config_paths_parses_as_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
-
-    assert parser.config_paths == ['myconfig', 'otherconfig']
-
-
-def test_parse_arguments_supports_show_flag():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments('--config', 'myconfig', '--show')
-
-    assert parser.show
+    module.main()

tests/integration/config/test_generate.py

@@ -210,7 +210,7 @@ def test_generate_sample_configuration_does_not_raise():
     flexmock(module).should_receive('_comment_out_optional_configuration')
     flexmock(module).should_receive('write_configuration')
 
-    module.generate_sample_configuration(None, 'dest.yaml', 'schema.yaml')
+    module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml')
 
 
 def test_generate_sample_configuration_with_source_filename_does_not_raise():
@@ -225,4 +225,17 @@ def test_generate_sample_configuration_with_source_filename_does_not_raise():
     flexmock(module).should_receive('_comment_out_optional_configuration')
     flexmock(module).should_receive('write_configuration')
 
-    module.generate_sample_configuration('source.yaml', 'dest.yaml', 'schema.yaml')
+    module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml')
+
+
+def test_generate_sample_configuration_with_dry_run_does_not_write_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module.yaml).should_receive('round_trip_load')
+    flexmock(module).should_receive('_schema_to_sample_configuration')
+    flexmock(module).should_receive('merge_source_configuration_into_destination')
+    flexmock(module).should_receive('render_configuration')
+    flexmock(module).should_receive('_comment_out_optional_configuration')
+    flexmock(module).should_receive('write_configuration').never()
+
+    module.generate_sample_configuration(True, None, 'dest.yaml', 'schema.yaml')

tests/unit/actions/config/test_bootstrap.py

@@ -0,0 +1,128 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.actions.config import bootstrap as module
+
+
+def test_get_config_paths_returns_list_of_config_paths():
+    bootstrap_arguments = flexmock(
+        borgmatic_source_directory=None,
+        repository='repo',
+        archive='archive',
+    )
+    global_arguments = flexmock(
+        dry_run=False,
+    )
+    local_borg_version = flexmock()
+    extract_process = flexmock(
+        stdout=flexmock(
+            read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
+        ),
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        extract_process
+    )
+    flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+    assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [
+        '/borgmatic/config.yaml'
+    ]
+
+
+def test_get_config_paths_with_missing_manifest_raises_value_error():
+    bootstrap_arguments = flexmock(
+        borgmatic_source_directory=None,
+        repository='repo',
+        archive='archive',
+    )
+    global_arguments = flexmock(
+        dry_run=False,
+    )
+    local_borg_version = flexmock()
+    extract_process = flexmock(stdout=flexmock(read=lambda: ''))
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        extract_process
+    )
+    flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+
+    with pytest.raises(ValueError):
+        module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version)
+
+
+def test_get_config_paths_with_broken_json_raises_value_error():
+    bootstrap_arguments = flexmock(
+        borgmatic_source_directory=None,
+        repository='repo',
+        archive='archive',
+    )
+    global_arguments = flexmock(
+        dry_run=False,
+    )
+    local_borg_version = flexmock()
+    extract_process = flexmock(
+        stdout=flexmock(read=lambda: '{"config_paths": ["/oops'),
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        extract_process
+    )
+    flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+
+    with pytest.raises(ValueError):
+        module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version)
+
+
+def test_get_config_paths_with_json_missing_key_raises_value_error():
+    bootstrap_arguments = flexmock(
+        borgmatic_source_directory=None,
+        repository='repo',
+        archive='archive',
+    )
+    global_arguments = flexmock(
+        dry_run=False,
+    )
+    local_borg_version = flexmock()
+    extract_process = flexmock(
+        stdout=flexmock(read=lambda: '{}'),
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        extract_process
+    )
+    flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+
+    with pytest.raises(ValueError):
+        module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version)
+
+
+def test_run_bootstrap_does_not_raise():
+    bootstrap_arguments = flexmock(
+        repository='repo',
+        archive='archive',
+        destination='dest',
+        strip_components=1,
+        progress=False,
+        borgmatic_source_directory='/borgmatic',
+    )
+    global_arguments = flexmock(
+        dry_run=False,
+    )
+    local_borg_version = flexmock()
+    extract_process = flexmock(
+        stdout=flexmock(
+            read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
+        ),
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        extract_process
+    ).twice()
+    flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+
+    module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)

tests/unit/actions/config/test_generate.py

@@ -0,0 +1,39 @@
+from flexmock import flexmock
+
+from borgmatic.actions.config import generate as module
+
+
+def test_run_generate_does_not_raise():
+    generate_arguments = flexmock(
+        source_filename=None,
+        destination_filename='destination.yaml',
+        overwrite=False,
+    )
+    global_arguments = flexmock(dry_run=False)
+    flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration')
+
+    module.run_generate(generate_arguments, global_arguments)
+
+
+def test_run_generate_with_dry_run_does_not_raise():
+    generate_arguments = flexmock(
+        source_filename=None,
+        destination_filename='destination.yaml',
+        overwrite=False,
+    )
+    global_arguments = flexmock(dry_run=True)
+    flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration')
+
+    module.run_generate(generate_arguments, global_arguments)
+
+
+def test_run_generate_with_source_filename_does_not_raise():
+    generate_arguments = flexmock(
+        source_filename='source.yaml',
+        destination_filename='destination.yaml',
+        overwrite=False,
+    )
+    global_arguments = flexmock(dry_run=False)
+    flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration')
+
+    module.run_generate(generate_arguments, global_arguments)

tests/unit/actions/config/test_validate.py

@@ -0,0 +1,17 @@
+from flexmock import flexmock
+
+from borgmatic.actions.config import validate as module
+
+
+def test_run_validate_does_not_raise():
+    validate_arguments = flexmock(show=False)
+    flexmock(module.borgmatic.config.generate).should_receive('render_configuration')
+
+    module.run_validate(validate_arguments, flexmock())
+
+
+def test_run_validate_with_show_does_not_raise():
+    validate_arguments = flexmock(show=True)
+    flexmock(module.borgmatic.config.generate).should_receive('render_configuration')
+
+    module.run_validate(validate_arguments, {'test.yaml': flexmock(), 'other.yaml': flexmock()})

tests/unit/actions/test_create.py

@@ -1,3 +1,5 @@
+import sys
+
 from flexmock import flexmock
 
 from borgmatic.actions import create as module
@@ -7,6 +9,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
+    flexmock(module).should_receive('create_borgmatic_manifest').once()
     flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({})
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
@@ -19,7 +22,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
         json=flexmock(),
         list_files=flexmock(),
     )
-    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
 
     list(
         module.run_create(
@@ -45,6 +48,7 @@ def test_run_create_runs_with_selected_repository():
         'repositories_match'
     ).once().and_return(True)
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
+    flexmock(module).should_receive('create_borgmatic_manifest').once()
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),
@@ -52,7 +56,7 @@ def test_run_create_runs_with_selected_repository():
         json=flexmock(),
         list_files=flexmock(),
     )
-    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
 
     list(
         module.run_create(
@@ -78,6 +82,7 @@ def test_run_create_bails_if_repository_does_not_match():
         'repositories_match'
     ).once().and_return(False)
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').never()
+    flexmock(module).should_receive('create_borgmatic_manifest').never()
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),
@@ -85,7 +90,7 @@ def test_run_create_bails_if_repository_does_not_match():
         json=flexmock(),
         list_files=flexmock(),
     )
-    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
 
     list(
         module.run_create(
@@ -103,3 +108,53 @@ def test_run_create_bails_if_repository_does_not_match():
             remote_path=None,
         )
     )
+
+
+def test_create_borgmatic_manifest_creates_manifest_file():
+    flexmock(module.os.path).should_receive('join').with_args(
+        module.borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY, 'bootstrap', 'manifest.json'
+    ).and_return('/home/user/.borgmatic/bootstrap/manifest.json')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os).should_receive('makedirs').and_return(True)
+
+    flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0')
+    flexmock(sys.modules['builtins']).should_receive('open').with_args(
+        '/home/user/.borgmatic/bootstrap/manifest.json', 'w'
+    ).and_return(
+        flexmock(
+            __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
+            __exit__=lambda *args: None,
+        )
+    )
+    flexmock(module.json).should_receive('dump').and_return(True).once()
+
+    module.create_borgmatic_manifest({}, 'test.yaml', False)
+
+
+def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_source_directory():
+    flexmock(module.os.path).should_receive('join').with_args(
+        '/borgmatic', 'bootstrap', 'manifest.json'
+    ).and_return('/borgmatic/bootstrap/manifest.json')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os).should_receive('makedirs').and_return(True)
+
+    flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0')
+    flexmock(sys.modules['builtins']).should_receive('open').with_args(
+        '/borgmatic/bootstrap/manifest.json', 'w'
+    ).and_return(
+        flexmock(
+            __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
+            __exit__=lambda *args: None,
+        )
+    )
+    flexmock(module.json).should_receive('dump').and_return(True).once()
+
+    module.create_borgmatic_manifest(
+        {'borgmatic_source_directory': '/borgmatic'}, 'test.yaml', False
+    )
+
+
+def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run():
+    flexmock(module.os.path).should_receive('expanduser').never()
+
+    module.create_borgmatic_manifest({}, 'test.yaml', True)

tests/unit/actions/test_restore.py

@@ -241,6 +241,7 @@ def test_run_restore_restores_each_database():
         archive_name=object,
         hook_name='postgresql_databases',
         database={'name': 'foo', 'schemas': None},
+        connection_params=object,
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -254,6 +255,7 @@ def test_run_restore_restores_each_database():
         archive_name=object,
         hook_name='postgresql_databases',
         database={'name': 'bar', 'schemas': None},
+        connection_params=object,
     ).once()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -264,7 +266,15 @@ def test_run_restore_restores_each_database():
         hooks=flexmock(),
         local_borg_version=flexmock(),
         restore_arguments=flexmock(
-            repository='repo', archive='archive', databases=flexmock(), schemas=None
+            repository='repo',
+            archive='archive',
+            databases=flexmock(),
+            schemas=None,
+            hostname=None,
+            port=None,
+            username=None,
+            password=None,
+            restore_path=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
@@ -337,6 +347,7 @@ def test_run_restore_restores_database_configured_with_all_name():
         archive_name=object,
         hook_name='postgresql_databases',
         database={'name': 'foo', 'schemas': None},
+        connection_params=object,
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -350,6 +361,7 @@ def test_run_restore_restores_database_configured_with_all_name():
         archive_name=object,
         hook_name='postgresql_databases',
         database={'name': 'bar', 'schemas': None},
+        connection_params=object,
     ).once()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -360,7 +372,15 @@ def test_run_restore_restores_database_configured_with_all_name():
         hooks=flexmock(),
         local_borg_version=flexmock(),
         restore_arguments=flexmock(
-            repository='repo', archive='archive', databases=flexmock(), schemas=None
+            repository='repo',
+            archive='archive',
+            databases=flexmock(),
+            schemas=None,
+            hostname=None,
+            port=None,
+            username=None,
+            password=None,
+            restore_path=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
@@ -411,6 +431,7 @@ def test_run_restore_skips_missing_database():
         archive_name=object,
         hook_name='postgresql_databases',
         database={'name': 'foo', 'schemas': None},
+        connection_params=object,
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -424,6 +445,7 @@ def test_run_restore_skips_missing_database():
         archive_name=object,
         hook_name='postgresql_databases',
         database={'name': 'bar', 'schemas': None},
+        connection_params=object,
     ).never()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -434,7 +456,15 @@ def test_run_restore_skips_missing_database():
         hooks=flexmock(),
         local_borg_version=flexmock(),
         restore_arguments=flexmock(
-            repository='repo', archive='archive', databases=flexmock(), schemas=None
+            repository='repo',
+            archive='archive',
+            databases=flexmock(),
+            schemas=None,
+            hostname=None,
+            port=None,
+            username=None,
+            password=None,
+            restore_path=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
@@ -479,6 +509,7 @@ def test_run_restore_restores_databases_from_different_hooks():
         archive_name=object,
         hook_name='postgresql_databases',
         database={'name': 'foo', 'schemas': None},
+        connection_params=object,
     ).once()
     flexmock(module).should_receive('restore_single_database').with_args(
         repository=object,
@@ -492,6 +523,7 @@ def test_run_restore_restores_databases_from_different_hooks():
         archive_name=object,
         hook_name='mysql_databases',
         database={'name': 'bar', 'schemas': None},
+        connection_params=object,
     ).once()
     flexmock(module).should_receive('ensure_databases_found')
 
@@ -502,7 +534,15 @@ def test_run_restore_restores_databases_from_different_hooks():
         hooks=flexmock(),
         local_borg_version=flexmock(),
         restore_arguments=flexmock(
-            repository='repo', archive='archive', databases=flexmock(), schemas=None
+            repository='repo',
+            archive='archive',
+            databases=flexmock(),
+            schemas=None,
+            hostname=None,
+            port=None,
+            username=None,
+            password=None,
+            restore_path=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),

tests/unit/borg/test_check.py

@@ -544,6 +544,34 @@ def test_upgrade_check_times_renames_old_check_paths_to_all():
     module.upgrade_check_times(flexmock(), flexmock())
 
 
+def test_upgrade_check_times_renames_data_check_paths_when_archives_paths_are_already_upgraded():
+    base_path = '~/.borgmatic/checks/1234'
+    flexmock(module).should_receive('make_check_time_path').with_args(
+        object, object, 'archives', 'all'
+    ).and_return(f'{base_path}/archives/all')
+    flexmock(module).should_receive('make_check_time_path').with_args(
+        object, object, 'data', 'all'
+    ).and_return(f'{base_path}/data/all')
+    flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
+        False
+    )
+    flexmock(module.os.path).should_receive('isfile').with_args(
+        f'{base_path}/archives.temp'
+    ).and_return(False)
+    flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
+        True
+    )
+    flexmock(module.os).should_receive('rename').with_args(
+        f'{base_path}/data', f'{base_path}/data.temp'
+    ).once()
+    flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/data').once()
+    flexmock(module.os).should_receive('rename').with_args(
+        f'{base_path}/data.temp', f'{base_path}/data/all'
+    ).once()
+
+    module.upgrade_check_times(flexmock(), flexmock())
+
+
 def test_upgrade_check_times_skips_missing_check_paths():
     flexmock(module).should_receive('make_check_time_path').and_return(
         '~/.borgmatic/checks/1234/archives/all'

tests/unit/borg/test_create.py

@@ -492,7 +492,7 @@ def test_create_archive_calls_borg_with_parameters():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -536,7 +536,7 @@ def test_create_archive_calls_borg_with_environment():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -582,7 +582,56 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
+    )
+
+
+def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sources_and_config_paths():
+    flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
+    flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
+    flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(
+        ('foo', 'bar', '/etc/borgmatic/config.yaml')
+    )
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('expand_directories').with_args(
+        ('foo', 'bar', '/etc/borgmatic/config.yaml')
+    ).and_return(('foo', 'bar', '/etc/borgmatic/config.yaml'))
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('pattern_root_directories').and_return([])
+    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
+    flexmock(module).should_receive('expand_home_directories').and_return(())
+    flexmock(module).should_receive('write_pattern_file').and_return(None)
+    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module).should_receive('ensure_files_readable')
+    flexmock(module).should_receive('make_pattern_flags').and_return(())
+    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{DEFAULT_ARCHIVE_NAME}',)
+    )
+    environment = {'BORG_THINGY': 'YUP'}
+    flexmock(module.environment).should_receive('make_environment').and_return(environment)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('/etc/borgmatic/config.yaml',),
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=environment,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository_path='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+        },
+        storage_config={},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']),
     )
 
 
@@ -628,7 +677,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -672,7 +721,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -713,7 +762,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
     )
 
@@ -758,7 +807,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -799,7 +848,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
     )
 
@@ -843,7 +892,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -889,7 +938,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stats=True,
     )
 
@@ -933,7 +982,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
         },
         storage_config={'checkpoint_interval': 600},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -976,7 +1025,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume
         },
         storage_config={'checkpoint_volume': 1024},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1019,7 +1068,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
         },
         storage_config={'chunker_params': '1,2,3,4'},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1062,7 +1111,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
         },
         storage_config={'compression': 'rle'},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1111,7 +1160,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_
         },
         storage_config={'upload_rate_limit': 100},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1157,7 +1206,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1201,7 +1250,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1251,7 +1300,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter(
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1305,7 +1354,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1361,7 +1410,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1416,7 +1465,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1471,7 +1520,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1515,7 +1564,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1558,7 +1607,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         local_path='borg1',
     )
 
@@ -1602,7 +1651,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         remote_path='borg1',
     )
 
@@ -1646,7 +1695,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
         },
         storage_config={'umask': 740},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1689,7 +1738,7 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=True),
+        global_arguments=flexmock(log_json=True, used_config_paths=[]),
     )
 
 
@@ -1732,7 +1781,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
         },
         storage_config={'lock_wait': 5},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1775,7 +1824,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stats=True,
     )
 
@@ -1819,7 +1868,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         list_files=True,
     )
 
@@ -1869,7 +1918,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         progress=True,
     )
 
@@ -1913,7 +1962,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         progress=True,
     )
 
@@ -1974,7 +2023,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         progress=True,
         stream_processes=processes,
     )
@@ -2039,7 +2088,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2107,7 +2156,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2172,7 +2221,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2213,7 +2262,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
     )
 
@@ -2256,7 +2305,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
         stats=True,
     )
@@ -2304,7 +2353,7 @@ def test_create_archive_with_source_directories_glob_expands():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2348,7 +2397,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2391,7 +2440,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2434,7 +2483,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
         },
         storage_config={'archive_name_format': 'ARCHIVE_NAME'},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2478,7 +2527,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
         },
         storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},  # noqa: FS003
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2522,7 +2571,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders():
         },
         storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},  # noqa: FS003
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2565,7 +2614,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
         },
         storage_config={'extra_borg_options': {'create': '--extra --options'}},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2626,7 +2675,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2652,7 +2701,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_
             },
             storage_config={},
             local_borg_version='1.2.3',
-            global_arguments=flexmock(log_json=False),
+            global_arguments=flexmock(log_json=False, used_config_paths=[]),
         )
 
 

tests/unit/commands/completion/test_actions.py

@@ -0,0 +1,7 @@
+from borgmatic.commands.completion import actions as module
+
+
+def test_upgrade_message_does_not_raise():
+    module.upgrade_message(
+        language='English', upgrade_command='read a lot', completion_file='your brain'
+    )

tests/unit/commands/completion/test_bash.py

@@ -0,0 +1,17 @@
+from flexmock import flexmock
+
+from borgmatic.commands.completion import bash as module
+
+
+def test_parser_flags_flattens_and_joins_flags():
+    assert (
+        module.parser_flags(
+            flexmock(
+                _actions=[
+                    flexmock(option_strings=['--foo', '--bar']),
+                    flexmock(option_strings=['--baz']),
+                ]
+            )
+        )
+        == '--foo --bar --baz'
+    )

tests/unit/commands/completion/test_fish.py

@@ -5,7 +5,7 @@ from typing import Tuple
 import pytest
 from flexmock import flexmock
 
-from borgmatic.commands import completion as module
+from borgmatic.commands.completion import fish as module
 
 OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required'])
 TestCase = Tuple[Action, OptionType]

tests/unit/commands/test_arguments.py

@@ -1,166 +1,431 @@
 import collections
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.commands import arguments as module
 
 
-def test_parse_subparser_arguments_consumes_subparser_arguments_before_subparser_name():
-    action_namespace = flexmock(foo=True)
-    subparsers = {
-        'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])),
-        'other': flexmock(),
-    }
+def test_get_subaction_parsers_with_no_subactions_returns_empty_result():
+    assert module.get_subaction_parsers(flexmock(_subparsers=None)) == {}
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(
-        ('--foo', 'true', 'action'), subparsers
+
+def test_get_subaction_parsers_with_subactions_returns_one_entry_per_subaction():
+    foo_parser = flexmock()
+    bar_parser = flexmock()
+    baz_parser = flexmock()
+
+    assert module.get_subaction_parsers(
+        flexmock(
+            _subparsers=flexmock(
+                _group_actions=(
+                    flexmock(choices={'foo': foo_parser, 'bar': bar_parser}),
+                    flexmock(choices={'baz': baz_parser}),
+                )
+            )
+        )
+    ) == {'foo': foo_parser, 'bar': bar_parser, 'baz': baz_parser}
+
+
+def test_get_subactions_for_actions_with_no_subactions_returns_empty_result():
+    assert module.get_subactions_for_actions({'action': flexmock(_subparsers=None)}) == {}
+
+
+def test_get_subactions_for_actions_with_subactions_returns_one_entry_per_action():
+    assert module.get_subactions_for_actions(
+        {
+            'action': flexmock(
+                _subparsers=flexmock(
+                    _group_actions=(
+                        flexmock(choices={'foo': flexmock(), 'bar': flexmock()}),
+                        flexmock(choices={'baz': flexmock()}),
+                    )
+                )
+            ),
+            'other': flexmock(
+                _subparsers=flexmock(_group_actions=(flexmock(choices={'quux': flexmock()}),))
+            ),
+        }
+    ) == {'action': ('foo', 'bar', 'baz'), 'other': ('quux',)}
+
+
+def test_omit_values_colliding_with_action_names_drops_action_names_that_have__been_parsed_as_values():
+    assert module.omit_values_colliding_with_action_names(
+        ('check', '--only', 'extract', '--some-list', 'borg'),
+        {'check': flexmock(only='extract', some_list=['borg'])},
+    ) == ('check', '--only', '--some-list')
+
+
+def test_parse_and_record_action_arguments_without_action_name_leaves_arguments_untouched():
+    unparsed_arguments = ('--foo', '--bar')
+    flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return(
+        unparsed_arguments
     )
 
-    assert arguments == {'action': action_namespace}
-    assert remaining_arguments == []
-
-
-def test_parse_subparser_arguments_consumes_subparser_arguments_after_subparser_name():
-    action_namespace = flexmock(foo=True)
-    subparsers = {
-        'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])),
-        'other': flexmock(),
-    }
-
-    arguments, remaining_arguments = module.parse_subparser_arguments(
-        ('action', '--foo', 'true'), subparsers
+    assert (
+        module.parse_and_record_action_arguments(
+            unparsed_arguments, flexmock(), flexmock(), 'action'
+        )
+        == unparsed_arguments
     )
 
-    assert arguments == {'action': action_namespace}
-    assert remaining_arguments == []
+
+def test_parse_and_record_action_arguments_updates_parsed_arguments_and_returns_remaining():
+    unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1')
+    other_parsed_arguments = flexmock()
+    parsed_arguments = {'other': other_parsed_arguments}
+    action_parsed_arguments = flexmock()
+    flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return(
+        unparsed_arguments
+    )
+    action_parser = flexmock()
+    flexmock(action_parser).should_receive('parse_known_args').and_return(
+        action_parsed_arguments, ('action', '--verbosity', '1')
+    )
+
+    assert module.parse_and_record_action_arguments(
+        unparsed_arguments, parsed_arguments, action_parser, 'action'
+    ) == ('--verbosity', '1')
+    assert parsed_arguments == {'other': other_parsed_arguments, 'action': action_parsed_arguments}
 
 
-def test_parse_subparser_arguments_consumes_subparser_arguments_with_alias():
+def test_parse_and_record_action_arguments_with_alias_updates_canonical_parsed_arguments():
+    unparsed_arguments = ('action', '--foo', '--bar', '--verbosity', '1')
+    other_parsed_arguments = flexmock()
+    parsed_arguments = {'other': other_parsed_arguments}
+    action_parsed_arguments = flexmock()
+    flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return(
+        unparsed_arguments
+    )
+    action_parser = flexmock()
+    flexmock(action_parser).should_receive('parse_known_args').and_return(
+        action_parsed_arguments, ('action', '--verbosity', '1')
+    )
+
+    assert module.parse_and_record_action_arguments(
+        unparsed_arguments, parsed_arguments, action_parser, 'action', canonical_name='doit'
+    ) == ('--verbosity', '1')
+    assert parsed_arguments == {'other': other_parsed_arguments, 'doit': action_parsed_arguments}
+
+
+def test_parse_and_record_action_arguments_with_borg_action_consumes_arguments_after_action_name():
+    unparsed_arguments = ('--verbosity', '1', 'borg', 'list')
+    parsed_arguments = {}
+    borg_parsed_arguments = flexmock(options=flexmock())
+    flexmock(module).should_receive('omit_values_colliding_with_action_names').and_return(
+        unparsed_arguments
+    )
+    borg_parser = flexmock()
+    flexmock(borg_parser).should_receive('parse_known_args').and_return(
+        borg_parsed_arguments, ('--verbosity', '1', 'borg', 'list')
+    )
+
+    assert module.parse_and_record_action_arguments(
+        unparsed_arguments,
+        parsed_arguments,
+        borg_parser,
+        'borg',
+    ) == ('--verbosity', '1')
+    assert parsed_arguments == {'borg': borg_parsed_arguments}
+    assert borg_parsed_arguments.options == ('list',)
+
+
+@pytest.mark.parametrize(
+    'arguments, expected',
+    [
+        # A global flag remaining from each parsed action.
+        (
+            (
+                ('--latest', 'archive', 'prune', 'extract', 'list', '--test-flag'),
+                ('--latest', 'archive', 'check', 'extract', 'list', '--test-flag'),
+                ('prune', 'check', 'list', '--test-flag'),
+                ('prune', 'check', 'extract', '--test-flag'),
+            ),
+            ('--test-flag',),
+        ),
+        # No global flags remaining.
+        (
+            (
+                ('--latest', 'archive', 'prune', 'extract', 'list'),
+                ('--latest', 'archive', 'check', 'extract', 'list'),
+                ('prune', 'check', 'list'),
+                ('prune', 'check', 'extract'),
+            ),
+            (),
+        ),
+        # No flags.
+        ((), ()),
+    ],
+)
+def test_get_unparsable_arguments_returns_remaining_arguments_that_no_action_can_parse(
+    arguments, expected
+):
+    assert module.get_unparsable_arguments(arguments) == expected
+
+
+def test_parse_arguments_for_actions_consumes_action_arguments_after_action_name():
     action_namespace = flexmock(foo=True)
-    action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, ['action']))
-    subparsers = {
-        'action': action_subparser,
-        '-a': action_subparser,
+    remaining = flexmock()
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: action_namespace}
+        )
+        or remaining
+    )
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {'action': flexmock(), 'other': flexmock()}
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
+
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('action', '--foo', 'true'), action_parsers, global_parser
+    )
+
+    assert arguments == {'global': global_namespace, 'action': action_namespace}
+    assert remaining_action_arguments == (remaining, ())
+
+
+def test_parse_arguments_for_actions_consumes_action_arguments_with_alias():
+    action_namespace = flexmock(foo=True)
+    remaining = flexmock()
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {canonical or action: action_namespace}
+        )
+        or remaining
+    )
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'action': flexmock(),
+        '-a': flexmock(),
         'other': flexmock(),
         '-o': flexmock(),
     }
-    flexmock(module).SUBPARSER_ALIASES = {'action': ['-a'], 'other': ['-o']}
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
+    flexmock(module).ACTION_ALIASES = {'action': ['-a'], 'other': ['-o']}
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(
-        ('-a', '--foo', 'true'), subparsers
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('-a', '--foo', 'true'), action_parsers, global_parser
     )
 
-    assert arguments == {'action': action_namespace}
-    assert remaining_arguments == []
+    assert arguments == {'global': global_namespace, 'action': action_namespace}
+    assert remaining_action_arguments == (remaining, ())
 
 
-def test_parse_subparser_arguments_consumes_multiple_subparser_arguments():
+def test_parse_arguments_for_actions_consumes_multiple_action_arguments():
     action_namespace = flexmock(foo=True)
     other_namespace = flexmock(bar=3)
-    subparsers = {
-        'action': flexmock(
-            parse_known_args=lambda arguments: (action_namespace, ['action', '--bar', '3'])
-        ),
-        'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])),
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: action_namespace if action == 'action' else other_namespace}
+        )
+        or ()
+    ).and_return(('other', '--bar', '3')).and_return('action', '--foo', 'true')
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'action': flexmock(),
+        'other': flexmock(),
     }
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(
-        ('action', '--foo', 'true', 'other', '--bar', '3'), subparsers
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('action', '--foo', 'true', 'other', '--bar', '3'), action_parsers, global_parser
     )
 
-    assert arguments == {'action': action_namespace, 'other': other_namespace}
-    assert remaining_arguments == []
+    assert arguments == {
+        'global': global_namespace,
+        'action': action_namespace,
+        'other': other_namespace,
+    }
+    assert remaining_action_arguments == ((), (), ())
 
 
-def test_parse_subparser_arguments_respects_command_line_action_ordering():
+def test_parse_arguments_for_actions_respects_command_line_action_ordering():
     other_namespace = flexmock()
     action_namespace = flexmock(foo=True)
-    subparsers = {
-        'action': flexmock(
-            parse_known_args=lambda arguments: (action_namespace, ['action', '--foo', 'true'])
-        ),
-        'other': flexmock(parse_known_args=lambda arguments: (other_namespace, ['other'])),
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: other_namespace if action == 'other' else action_namespace}
+        )
+        or ()
+    ).and_return(('action',)).and_return(('other', '--foo', 'true'))
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'action': flexmock(),
+        'other': flexmock(),
     }
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(
-        ('other', '--foo', 'true', 'action'), subparsers
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('other', '--foo', 'true', 'action'), action_parsers, global_parser
     )
 
     assert arguments == collections.OrderedDict(
-        [('other', other_namespace), ('action', action_namespace)]
+        [('other', other_namespace), ('action', action_namespace), ('global', global_namespace)]
     )
-    assert remaining_arguments == []
+    assert remaining_action_arguments == ((), (), ())
 
 
-def test_parse_subparser_arguments_applies_default_subparsers():
-    prune_namespace = flexmock()
-    compact_namespace = flexmock()
-    create_namespace = flexmock(progress=True)
-    check_namespace = flexmock()
-    subparsers = {
-        'prune': flexmock(
-            parse_known_args=lambda arguments: (prune_namespace, ['prune', '--progress'])
-        ),
-        'compact': flexmock(parse_known_args=lambda arguments: (compact_namespace, [])),
-        'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])),
-        'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])),
+def test_parse_arguments_for_actions_applies_default_action_parsers():
+    global_namespace = flexmock(config_paths=[])
+    namespaces = {
+        'global': global_namespace,
+        'prune': flexmock(),
+        'compact': flexmock(),
+        'create': flexmock(progress=True),
+        'check': flexmock(),
+    }
+
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: namespaces.get(action)}
+        )
+        or ()
+    ).and_return(())
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'prune': flexmock(),
+        'compact': flexmock(),
+        'create': flexmock(),
+        'check': flexmock(),
         'other': flexmock(),
     }
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(('--progress'), subparsers)
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('--progress'), action_parsers, global_parser
+    )
 
-    assert arguments == {
-        'prune': prune_namespace,
-        'compact': compact_namespace,
-        'create': create_namespace,
-        'check': check_namespace,
-    }
-    assert remaining_arguments == []
+    assert arguments == namespaces
+    assert remaining_action_arguments == ((), (), (), (), ())
 
 
-def test_parse_subparser_arguments_passes_through_unknown_arguments_before_subparser_name():
+def test_parse_arguments_for_actions_consumes_global_arguments():
     action_namespace = flexmock()
-    subparsers = {
-        'action': flexmock(
-            parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots'])
-        ),
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: action_namespace}
+        )
+        or ('--verbosity', 'lots')
+    )
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'action': flexmock(),
         'other': flexmock(),
     }
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(
-        ('--verbosity', 'lots', 'action'), subparsers
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('action', '--verbosity', 'lots'), action_parsers, global_parser
     )
 
-    assert arguments == {'action': action_namespace}
-    assert remaining_arguments == ['--verbosity', 'lots']
+    assert arguments == {'global': global_namespace, 'action': action_namespace}
+    assert remaining_action_arguments == (('--verbosity', 'lots'), ())
 
 
-def test_parse_subparser_arguments_passes_through_unknown_arguments_after_subparser_name():
+def test_parse_arguments_for_actions_passes_through_unknown_arguments_before_action_name():
     action_namespace = flexmock()
-    subparsers = {
-        'action': flexmock(
-            parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots'])
-        ),
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: action_namespace}
+        )
+        or ('--wtf', 'yes')
+    )
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'action': flexmock(),
         'other': flexmock(),
     }
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(
-        ('action', '--verbosity', 'lots'), subparsers
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('--wtf', 'yes', 'action'), action_parsers, global_parser
     )
 
-    assert arguments == {'action': action_namespace}
-    assert remaining_arguments == ['--verbosity', 'lots']
+    assert arguments == {'global': global_namespace, 'action': action_namespace}
+    assert remaining_action_arguments == (('--wtf', 'yes'), ())
 
 
-def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparsers():
+def test_parse_arguments_for_actions_passes_through_unknown_arguments_after_action_name():
+    action_namespace = flexmock()
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: action_namespace}
+        )
+        or ('--wtf', 'yes')
+    )
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'action': flexmock(),
+        'other': flexmock(),
+    }
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
+
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('action', '--wtf', 'yes'), action_parsers, global_parser
+    )
+
+    assert arguments == {'global': global_namespace, 'action': action_namespace}
+    assert remaining_action_arguments == (('--wtf', 'yes'), ())
+
+
+def test_parse_arguments_for_actions_with_borg_action_skips_other_action_parsers():
     action_namespace = flexmock(options=[])
-    subparsers = {
-        'borg': flexmock(parse_known_args=lambda arguments: (action_namespace, ['borg', 'list'])),
+    flexmock(module).should_receive('get_subaction_parsers').and_return({})
+    flexmock(module).should_receive('parse_and_record_action_arguments').replace_with(
+        lambda unparsed, parsed, parser, action, canonical=None: parsed.update(
+            {action: action_namespace}
+        )
+        or ()
+    ).and_return(())
+    flexmock(module).should_receive('get_subactions_for_actions').and_return({})
+    action_parsers = {
+        'borg': flexmock(),
         'list': flexmock(),
     }
+    global_namespace = flexmock(config_paths=[])
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((global_namespace, ()))
 
-    arguments, remaining_arguments = module.parse_subparser_arguments(('borg', 'list'), subparsers)
+    arguments, remaining_action_arguments = module.parse_arguments_for_actions(
+        ('borg', 'list'), action_parsers, global_parser
+    )
 
-    assert arguments == {'borg': action_namespace}
-    assert arguments['borg'].options == ['list']
-    assert remaining_arguments == []
+    assert arguments == {'global': global_namespace, 'borg': action_namespace}
+    assert remaining_action_arguments == ((), ())
+
+
+def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified():
+    flexmock(module).should_receive('get_subaction_parsers').and_return({'bootstrap': [flexmock()]})
+    flexmock(module).should_receive('parse_and_record_action_arguments').and_return(flexmock())
+    flexmock(module).should_receive('get_subactions_for_actions').and_return(
+        {'config': ['bootstrap']}
+    )
+    action_parsers = {'config': flexmock()}
+    global_parser = flexmock()
+    global_parser.should_receive('parse_known_args').and_return((flexmock(), ()))
+
+    with pytest.raises(ValueError):
+        module.parse_arguments_for_actions(('config',), action_parsers, global_parser)

tests/unit/commands/test_borgmatic.py

@@ -877,7 +877,7 @@ def test_load_configurations_collects_parsed_configurations_and_logs():
     configs, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml')))
 
     assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration}
-    assert logs == test_expected_logs + other_expected_logs
+    assert set(logs) >= set(test_expected_logs + other_expected_logs)
 
 
 def test_load_configurations_logs_warning_for_permission_error():
@@ -886,7 +886,7 @@ def test_load_configurations_logs_warning_for_permission_error():
     configs, logs = tuple(module.load_configurations(('test.yaml',)))
 
     assert configs == {}
-    assert {log.levelno for log in logs} == {logging.WARNING}
+    assert max(log.levelno for log in logs) == logging.WARNING
 
 
 def test_load_configurations_logs_critical_for_parse_error():
@@ -895,7 +895,7 @@ def test_load_configurations_logs_critical_for_parse_error():
     configs, logs = tuple(module.load_configurations(('test.yaml',)))
 
     assert configs == {}
-    assert {log.levelno for log in logs} == {logging.CRITICAL}
+    assert max(log.levelno for log in logs) == logging.CRITICAL
 
 
 def test_log_record_does_not_raise():
@@ -962,6 +962,140 @@ def test_get_local_path_without_local_path_defaults_to_borg():
     assert module.get_local_path({'test.yaml': {'location': {}}}) == 'borg'
 
 
+def test_collect_highlander_action_summary_logs_info_for_success_with_bootstrap():
+    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap')
+    arguments = {
+        'bootstrap': flexmock(repository='repo'),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False
+        )
+    )
+    assert {log.levelno for log in logs} == {logging.ANSWER}
+
+
+def test_collect_highlander_action_summary_logs_error_on_bootstrap_failure():
+    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise(
+        ValueError
+    )
+    arguments = {
+        'bootstrap': flexmock(repository='repo'),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False
+        )
+    )
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_collect_highlander_action_summary_logs_error_on_bootstrap_local_borg_version_failure():
+    flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError)
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').never()
+    arguments = {
+        'bootstrap': flexmock(repository='repo'),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False
+        )
+    )
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_collect_highlander_action_summary_logs_info_for_success_with_generate():
+    flexmock(module.borgmatic.actions.config.generate).should_receive('run_generate')
+    arguments = {
+        'generate': flexmock(destination='test.yaml'),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False
+        )
+    )
+    assert {log.levelno for log in logs} == {logging.ANSWER}
+
+
+def test_collect_highlander_action_summary_logs_error_on_generate_failure():
+    flexmock(module.borgmatic.actions.config.generate).should_receive('run_generate').and_raise(
+        ValueError
+    )
+    arguments = {
+        'generate': flexmock(destination='test.yaml'),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False
+        )
+    )
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_collect_highlander_action_summary_logs_info_for_success_with_validate():
+    flexmock(module.borgmatic.actions.config.validate).should_receive('run_validate')
+    arguments = {
+        'validate': flexmock(),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False
+        )
+    )
+    assert {log.levelno for log in logs} == {logging.ANSWER}
+
+
+def test_collect_highlander_action_summary_logs_error_on_validate_parse_failure():
+    flexmock(module.borgmatic.actions.config.validate).should_receive('run_validate')
+    arguments = {
+        'validate': flexmock(),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=True
+        )
+    )
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_collect_highlander_action_summary_logs_error_on_run_validate_failure():
+    flexmock(module.borgmatic.actions.config.validate).should_receive('run_validate').and_raise(
+        ValueError
+    )
+    arguments = {
+        'validate': flexmock(),
+        'global': flexmock(dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_highlander_action_summary_logs(
+            {'test.yaml': {}}, arguments=arguments, configuration_parse_errors=False
+        )
+    )
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
 def test_collect_configuration_run_summary_logs_info_for_success():
     flexmock(module.command).should_receive('execute_hook').never()
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')

tests/unit/config/test_collect.py

@@ -29,15 +29,6 @@ def test_get_default_config_paths_does_not_expand_home_when_false():
     assert '$HOME/.config/borgmatic/config.yaml' in config_paths
 
 
-def test_collect_config_filenames_collects_given_files():
-    config_paths = ('config.yaml', 'other.yaml')
-    flexmock(module.os.path).should_receive('isdir').and_return(False)
-
-    config_filenames = tuple(module.collect_config_filenames(config_paths))
-
-    assert config_filenames == config_paths
-
-
 def test_collect_config_filenames_collects_yml_file_endings():
     config_paths = ('config.yaml', '/etc/borgmatic.d')
     mock_path = flexmock(module.os.path)
@@ -45,13 +36,14 @@ def test_collect_config_filenames_collects_yml_file_endings():
     mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
     flexmock(module.os).should_receive('access').and_return(True)
     flexmock(module.os).should_receive('listdir')
     flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml'])
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
-    assert config_filenames == ('config.yaml', '/etc/borgmatic.d/foo.yml')
+    assert config_filenames == ('/config.yaml', '/etc/borgmatic.d/foo.yml')
 
 
 def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
@@ -63,6 +55,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
     flexmock(module.os).should_receive('access').and_return(True)
     flexmock(module.os).should_receive('listdir')
     flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
@@ -72,7 +65,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
     assert config_filenames == (
-        'config.yaml',
+        '/config.yaml',
         '/etc/borgmatic.d/foo.yaml',
         '/etc/borgmatic.d/baz.yaml',
     )
@@ -86,6 +79,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
     flexmock(module.os).should_receive('access').and_return(True)
     flexmock(module.os).should_receive('listdir')
     flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
@@ -103,13 +97,14 @@ def test_collect_config_filenames_skips_permission_denied_directories():
     mock_path.should_receive('exists').and_return(True)
     mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
     flexmock(module.os).should_receive('access').and_return(False)
     flexmock(module.os).should_receive('listdir')
     flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['config.yaml'])
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
-    assert config_filenames == ('config.yaml',)
+    assert config_filenames == ('/config.yaml',)
 
 
 def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist():
@@ -119,10 +114,11 @@ def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does
     mock_path.should_receive('exists').with_args('/etc/borgmatic/config.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic/config.yaml').and_return(True)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
-    assert config_filenames == ('config.yaml',)
+    assert config_filenames == ('/config.yaml',)
 
 
 def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
@@ -132,10 +128,11 @@ def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist
     mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False)
     mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
-    assert config_filenames == ('config.yaml',)
+    assert config_filenames == ('/config.yaml',)
 
 
 def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_does_not_exist():
@@ -145,10 +142,11 @@ def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_
     mock_path.should_receive('exists').with_args('/etc/../etc/borgmatic.d').and_return(False)
     mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/../etc/borgmatic.d').and_return(True)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
-    assert config_filenames == ('config.yaml',)
+    assert config_filenames == ('/config.yaml',)
 
 
 def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist():
@@ -158,7 +156,8 @@ def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist(
     mock_path.should_receive('exists').with_args('/my/directory').and_return(False)
     mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/my/directory').and_return(True)
+    mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
-    assert config_filenames == config_paths
+    assert config_filenames == ('/config.yaml', '/my/directory')

tests/unit/config/test_convert.py

@@ -1,126 +0,0 @@
-import os
-from collections import OrderedDict, defaultdict, namedtuple
-
-import pytest
-from flexmock import flexmock
-
-from borgmatic.config import convert as module
-
-Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
-
-
-def test_convert_section_generates_integer_value_for_integer_type_in_schema():
-    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    source_section_config = OrderedDict([('check_last', '3')])
-    section_schema = {'type': 'object', 'properties': {'check_last': {'type': 'integer'}}}
-
-    destination_config = module._convert_section(source_section_config, section_schema)
-
-    assert destination_config == OrderedDict([('check_last', 3)])
-
-
-def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
-    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module.generate).should_receive('add_comments_to_configuration_object')
-    source_config = Parsed_config(
-        location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
-        storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
-        retention=OrderedDict([('keep_daily', 7)]),
-        consistency=OrderedDict([('checks', 'repository')]),
-    )
-    source_excludes = ['/var']
-    schema = {
-        'type': 'object',
-        'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}),
-    }
-
-    destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)
-
-    assert destination_config == OrderedDict(
-        [
-            (
-                'location',
-                OrderedDict(
-                    [
-                        ('source_directories', ['/home']),
-                        ('repositories', ['hostname.borg']),
-                        ('exclude_patterns', ['/var']),
-                    ]
-                ),
-            ),
-            ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
-            ('retention', OrderedDict([('keep_daily', 7)])),
-            ('consistency', OrderedDict([('checks', ['repository'])])),
-        ]
-    )
-
-
-def test_convert_legacy_parsed_config_splits_space_separated_values():
-    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module.generate).should_receive('add_comments_to_configuration_object')
-    source_config = Parsed_config(
-        location=OrderedDict(
-            [('source_directories', '/home /etc'), ('repository', 'hostname.borg')]
-        ),
-        storage=OrderedDict(),
-        retention=OrderedDict(),
-        consistency=OrderedDict([('checks', 'repository archives')]),
-    )
-    source_excludes = ['/var']
-    schema = {
-        'type': 'object',
-        'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}),
-    }
-
-    destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)
-
-    assert destination_config == OrderedDict(
-        [
-            (
-                'location',
-                OrderedDict(
-                    [
-                        ('source_directories', ['/home', '/etc']),
-                        ('repositories', ['hostname.borg']),
-                        ('exclude_patterns', ['/var']),
-                    ]
-                ),
-            ),
-            ('storage', OrderedDict()),
-            ('retention', OrderedDict()),
-            ('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
-        ]
-    )
-
-
-def test_guard_configuration_upgraded_raises_when_only_source_config_present():
-    flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
-    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
-    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
-
-    with pytest.raises(module.Legacy_configuration_not_upgraded):
-        module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
-
-
-def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present():
-    flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
-    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
-    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
-
-    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
-
-
-def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present():
-    flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
-    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
-    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
-
-    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
-
-
-def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present():
-    flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
-    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
-    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
-
-    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))

tests/unit/hooks/test_mongodb.py

@@ -171,7 +171,17 @@ def test_restore_database_dump_runs_mongorestore():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -185,7 +195,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
 
     with pytest.raises(ValueError):
         module.restore_database_dump(
-            database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+            database_config,
+            'test.yaml',
+            {},
+            dry_run=False,
+            extract_process=flexmock(),
+            connection_params={
+                'hostname': None,
+                'port': None,
+                'username': None,
+                'password': None,
+            },
         )
 
 
@@ -215,7 +235,17 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -253,7 +283,129 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
+    database_config = [
+        {
+            'name': 'foo',
+            'username': 'mongo',
+            'password': 'trustsome1',
+            'authentication_database': 'admin',
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+            'restore_username': 'restoreusername',
+            'restore_password': 'restorepassword',
+            'schemas': None,
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        [
+            'mongorestore',
+            '--archive',
+            '--drop',
+            '--db',
+            'foo',
+            '--host',
+            'clihost',
+            '--port',
+            'cliport',
+            '--username',
+            'cliusername',
+            '--password',
+            'clipassword',
+            '--authenticationDatabase',
+            'admin',
+        ],
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': 'clihost',
+            'port': 'cliport',
+            'username': 'cliusername',
+            'password': 'clipassword',
+        },
+    )
+
+
+def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
+    database_config = [
+        {
+            'name': 'foo',
+            'username': 'mongo',
+            'password': 'trustsome1',
+            'authentication_database': 'admin',
+            'schemas': None,
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+            'restore_username': 'restoreuser',
+            'restore_password': 'restorepass',
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        [
+            'mongorestore',
+            '--archive',
+            '--drop',
+            '--db',
+            'foo',
+            '--host',
+            'restorehost',
+            '--port',
+            'restoreport',
+            '--username',
+            'restoreuser',
+            '--password',
+            'restorepass',
+            '--authenticationDatabase',
+            'admin',
+        ],
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -271,7 +423,17 @@ def test_restore_database_dump_runs_mongorestore_with_options():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -299,7 +461,17 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -317,7 +489,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -329,7 +511,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=True,
+        extract_process=flexmock(),
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -346,5 +538,15 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=None
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=None,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )

tests/unit/hooks/test_mysql.py

@@ -392,7 +392,17 @@ def test_restore_database_dump_runs_mysql_to_restore():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -404,7 +414,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
 
     with pytest.raises(ValueError):
         module.restore_database_dump(
-            database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+            database_config,
+            'test.yaml',
+            {},
+            dry_run=False,
+            extract_process=flexmock(),
+            connection_params={
+                'hostname': None,
+                'port': None,
+                'username': None,
+                'password': None,
+            },
         )
 
 
@@ -421,7 +441,17 @@ def test_restore_database_dump_runs_mysql_with_options():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -447,7 +477,17 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -464,7 +504,115 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
+    database_config = [
+        {
+            'name': 'foo',
+            'username': 'root',
+            'password': 'trustsome1',
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+            'restore_username': 'restoreusername',
+            'restore_password': 'restorepassword',
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mysql',
+            '--batch',
+            '--host',
+            'clihost',
+            '--port',
+            'cliport',
+            '--protocol',
+            'tcp',
+            '--user',
+            'cliusername',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'MYSQL_PWD': 'clipassword'},
+    ).once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': 'clihost',
+            'port': 'cliport',
+            'username': 'cliusername',
+            'password': 'clipassword',
+        },
+    )
+
+
+def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
+    database_config = [
+        {
+            'name': 'foo',
+            'username': 'root',
+            'password': 'trustsome1',
+            'hostname': 'dbhost',
+            'port': 'dbport',
+            'restore_username': 'restoreuser',
+            'restore_password': 'restorepass',
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mysql',
+            '--batch',
+            '--host',
+            'restorehost',
+            '--port',
+            'restoreport',
+            '--protocol',
+            'tcp',
+            '--user',
+            'restoreuser',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'MYSQL_PWD': 'restorepass'},
+    ).once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -474,5 +622,15 @@ def test_restore_database_dump_with_dry_run_skips_restore():
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=True,
+        extract_process=flexmock(),
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )

tests/unit/hooks/test_postgresql.py

@@ -6,6 +6,50 @@ from flexmock import flexmock
 from borgmatic.hooks import postgresql as module
 
 
+def test_make_extra_environment_maps_options_to_environment():
+    database = {
+        'name': 'foo',
+        'password': 'pass',
+        'ssl_mode': 'require',
+        'ssl_cert': 'cert.crt',
+        'ssl_key': 'key.key',
+        'ssl_root_cert': 'root.crt',
+        'ssl_crl': 'crl.crl',
+    }
+    expected = {
+        'PGPASSWORD': 'pass',
+        'PGSSLMODE': 'require',
+        'PGSSLCERT': 'cert.crt',
+        'PGSSLKEY': 'key.key',
+        'PGSSLROOTCERT': 'root.crt',
+        'PGSSLCRL': 'crl.crl',
+    }
+
+    extra_env = module.make_extra_environment(database)
+
+    assert extra_env == expected
+
+
+def test_make_extra_environment_with_cli_password_sets_correct_password():
+    database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
+
+    extra = module.make_extra_environment(
+        database, restore_connection_params={'password': 'clipassword'}
+    )
+
+    assert extra['PGPASSWORD'] == 'clipassword'
+
+
+def test_make_extra_environment_without_cli_password_or_configured_password_does_not_set_password():
+    database = {'name': 'foo'}
+
+    extra = module.make_extra_environment(
+        database, restore_connection_params={'username': 'someone'}
+    )
+
+    assert 'PGPASSWORD' not in extra
+
+
 def test_database_names_to_dump_passes_through_individual_database_name():
     database = {'name': 'foo'}
 
@@ -301,29 +345,6 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
-def test_make_extra_environment_maps_options_to_environment():
-    database = {
-        'name': 'foo',
-        'password': 'pass',
-        'ssl_mode': 'require',
-        'ssl_cert': 'cert.crt',
-        'ssl_key': 'key.key',
-        'ssl_root_cert': 'root.crt',
-        'ssl_crl': 'crl.crl',
-    }
-    expected = {
-        'PGPASSWORD': 'pass',
-        'PGSSLMODE': 'require',
-        'PGSSLCERT': 'cert.crt',
-        'PGSSLKEY': 'key.key',
-        'PGSSLROOTCERT': 'root.crt',
-        'PGSSLCRL': 'crl.crl',
-    }
-
-    extra_env = module.make_extra_environment(database)
-    assert extra_env == expected
-
-
 def test_dump_databases_runs_pg_dump_with_directory_format():
     databases = [{'name': 'foo', 'format': 'directory'}]
     flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
@@ -479,7 +500,17 @@ def test_restore_database_dump_runs_pg_restore():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -494,7 +525,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
 
     with pytest.raises(ValueError):
         module.restore_database_dump(
-            database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+            database_config,
+            'test.yaml',
+            {},
+            dry_run=False,
+            extract_process=flexmock(),
+            connection_params={
+                'restore_hostname': None,
+                'restore_port': None,
+                'restore_username': None,
+                'restore_password': None,
+            },
         )
 
 
@@ -545,7 +586,17 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -594,7 +645,173 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
+def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
+    database_config = [
+        {
+            'name': 'foo',
+            'hostname': 'database.example.org',
+            'port': 5433,
+            'username': 'postgres',
+            'password': 'trustsome1',
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+            'restore_username': 'restoreusername',
+            'restore_password': 'restorepassword',
+            'schemas': None,
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_extra_environment').and_return(
+        {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}
+    )
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+            '--host',
+            'clihost',
+            '--port',
+            'cliport',
+            '--username',
+            'cliusername',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'psql',
+            '--no-password',
+            '--no-psqlrc',
+            '--quiet',
+            '--host',
+            'clihost',
+            '--port',
+            'cliport',
+            '--username',
+            'cliusername',
+            '--dbname',
+            'foo',
+            '--command',
+            'ANALYZE',
+        ),
+        extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
+    ).once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': 'clihost',
+            'port': 'cliport',
+            'username': 'cliusername',
+            'password': 'clipassword',
+        },
+    )
+
+
+def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
+    database_config = [
+        {
+            'name': 'foo',
+            'hostname': 'database.example.org',
+            'port': 5433,
+            'username': 'postgres',
+            'password': 'trustsome1',
+            'schemas': None,
+            'restore_hostname': 'restorehost',
+            'restore_port': 'restoreport',
+            'restore_username': 'restoreusername',
+            'restore_password': 'restorepassword',
+        }
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_extra_environment').and_return(
+        {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}
+    )
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+            '--host',
+            'restorehost',
+            '--port',
+            'restoreport',
+            '--username',
+            'restoreusername',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'psql',
+            '--no-password',
+            '--no-psqlrc',
+            '--quiet',
+            '--host',
+            'restorehost',
+            '--port',
+            'restoreport',
+            '--username',
+            'restoreusername',
+            '--dbname',
+            'foo',
+            '--command',
+            'ANALYZE',
+        ),
+        extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
+    ).once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -644,7 +861,17 @@ def test_restore_database_dump_runs_pg_restore_with_options():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -672,7 +899,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -705,7 +942,17 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -759,7 +1006,17 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -772,7 +1029,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=True,
+        extract_process=flexmock(),
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -813,7 +1080,17 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=None
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=None,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )
 
 
@@ -858,5 +1135,15 @@ def test_restore_database_dump_with_schemas_restores_schemas():
     ).once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=None
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=None,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
     )

tests/unit/hooks/test_sqlite.py

@@ -1,3 +1,5 @@
+import logging
+
 import pytest
 from flexmock import flexmock
 
@@ -94,12 +96,81 @@ def test_restore_database_dump_restores_database():
     database_config = [{'path': '/path/to/database', 'name': 'database'}]
     extract_process = flexmock(stdout=flexmock())
 
-    flexmock(module).should_receive('execute_command_with_processes').once()
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'sqlite3',
+            '/path/to/database',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
 
     flexmock(module.os).should_receive('remove').once()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={'restore_path': None},
+    )
+
+
+def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
+    database_config = [
+        {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'sqlite3',
+            'cli/path/to/database',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    flexmock(module.os).should_receive('remove').once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={'restore_path': 'cli/path/to/database'},
+    )
+
+
+def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
+    database_config = [
+        {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'sqlite3',
+            'config/path/to/database',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+    ).once()
+
+    flexmock(module.os).should_receive('remove').once()
+
+    module.restore_database_dump(
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={'restore_path': None},
     )
 
 
@@ -111,7 +182,12 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run():
     flexmock(module.os).should_receive('remove').never()
 
     module.restore_database_dump(
-        database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process
+        database_config,
+        'test.yaml',
+        {},
+        dry_run=True,
+        extract_process=extract_process,
+        connection_params={'restore_path': None},
     )
 
 
@@ -121,5 +197,10 @@ def test_restore_database_dump_raises_error_if_database_config_is_invalid():
 
     with pytest.raises(ValueError):
         module.restore_database_dump(
-            database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+            database_config,
+            'test.yaml',
+            {},
+            dry_run=False,
+            extract_process=extract_process,
+            connection_params={'restore_path': None},
         )