Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/737

NEWS

@@ -1,3 +1,19 @@
+1.8.1
+ * #326: Add documentation for restoring a database to an alternate host:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-to-an-alternate-host
+ * #697: Add documentation for "bootstrap" action:
+   https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#extract-the-configuration-files-used-to-create-an-archive
+ * #725: Add "store_config_files" option for disabling the automatic backup of configuration files
+   used by the "config bootstrap" action.
+ * #728: Fix for "prune" action error when using the "keep_exclude_tags" option.
+ * #730: Fix for Borg's interactive prompt on the "check --repair" action automatically getting
+   answered "NO" even when the "check_i_know_what_i_am_doing" option isn't set.
+ * #732: Include multiple configuration files with a single "!include". See the documentation for
+   more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#multiple-merge-includes
+ * #734: Omit "--glob-archives" or "--match-archives" Borg flag when its value would be "*" (meaning
+   all archives).
+
 1.8.0
  * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting
    repository/archive into the resulting Borg command-line, pass repository to Borg via an

borgmatic/actions/create.py

@@ -91,7 +91,10 @@ def run_create(
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         global_arguments.dry_run,
     )
-    create_borgmatic_manifest(config, global_arguments.used_config_paths, global_arguments.dry_run)
+    if config.get('store_config_files', True):
+        create_borgmatic_manifest(
+            config, global_arguments.used_config_paths, global_arguments.dry_run
+        )
     stream_processes = [process for processes in active_dumps.values() for process in processes]
 
     json_output = borgmatic.borg.create.create_archive(

borgmatic/borg/check.py

@@ -142,7 +142,7 @@ def filter_checks_on_frequency(
         if datetime.datetime.now() < check_time + frequency_delta:
             remaining = check_time + frequency_delta - datetime.datetime.now()
             logger.info(
-                f'Skipping {check} check due to configured frequency; {remaining} until next check'
+                f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
             )
             filtered_checks.remove(check)
 

borgmatic/borg/create.py

@@ -354,7 +354,11 @@ def create_archive(
             expand_directories(
                 tuple(config.get('source_directories', ()))
                 + borgmatic_source_directories
-                + tuple(global_arguments.used_config_paths)
+                + tuple(
+                    global_arguments.used_config_paths
+                    if config.get('store_config_files', True)
+                    else ()
+                )
             )
         ),
         additional_directory_devices=map_directories_to_devices(

borgmatic/borg/environment.py

@@ -38,14 +38,16 @@ def make_environment(config):
         option_name,
         environment_variable_name,
     ) in DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE.items():
-        value = config.get(option_name, False)
-        environment[environment_variable_name] = 'yes' if value else 'no'
+        value = config.get(option_name)
+        if value is not None:
+            environment[environment_variable_name] = 'yes' if value else 'no'
 
     for (
         option_name,
         environment_variable_name,
     ) in DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE.items():
-        value = config.get(option_name, False)
-        environment[environment_variable_name] = 'YES' if value else 'NO'
+        value = config.get(option_name)
+        if value is not None:
+            environment[environment_variable_name] = 'YES' if value else 'NO'
 
     return environment

borgmatic/borg/flags.py

@@ -77,6 +77,9 @@ def make_match_archives_flags(match_archives, archive_name_format, local_borg_ve
 
     derived_match_archives = re.sub(r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format)
 
+    if derived_match_archives == '*':
+        return ()
+
     if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
         return ('--match-archives', f'sh:{derived_match_archives}')
     else:

borgmatic/borg/prune.py

@@ -26,7 +26,7 @@ def make_prune_flags(config, local_borg_version):
     flag_pairs = (
         ('--' + option_name.replace('_', '-'), str(value))
         for option_name, value in config.items()
-        if option_name.startswith('keep_')
+        if option_name.startswith('keep_') and option_name != 'keep_exclude_tags'
     )
     prefix = config.get('prefix')
 

borgmatic/commands/arguments.py

@@ -331,9 +331,8 @@ def make_parsers():
 
     global_plus_action_parser = ArgumentParser(
         description='''
-            Simple, configuration-driven backup software for servers and workstations. If none of
-            the action options are given, then borgmatic defaults to: create, prune, compact, and
-            check.
+            Simple, configuration-driven backup software for servers and workstations. If no actions
+            are given, then borgmatic defaults to: create, prune, compact, and check.
             ''',
         parents=[global_parser],
     )

borgmatic/config/load.py

@@ -1,6 +1,8 @@
 import functools
+import itertools
 import json
 import logging
+import operator
 import os
 
 import ruamel.yaml
@@ -8,34 +10,67 @@ import ruamel.yaml
 logger = logging.getLogger(__name__)
 
 
+def probe_and_include_file(filename, include_directories):
+    '''
+    Given a filename to include and a list of include directories to search for matching files,
+    probe for the file, load it, and return the loaded configuration as a data structure of nested
+    dicts, lists, etc.
+
+    Raise FileNotFoundError if the included file was not found.
+    '''
+    expanded_filename = os.path.expanduser(filename)
+
+    if os.path.isabs(expanded_filename):
+        return load_configuration(expanded_filename)
+
+    candidate_filenames = {
+        os.path.join(directory, expanded_filename) for directory in include_directories
+    }
+
+    for candidate_filename in candidate_filenames:
+        if os.path.exists(candidate_filename):
+            return load_configuration(candidate_filename)
+
+    raise FileNotFoundError(
+        f'Could not find include {filename} at {" or ".join(candidate_filenames)}'
+    )
+
+
 def include_configuration(loader, filename_node, include_directory):
     '''
-    Given a ruamel.yaml.loader.Loader, a ruamel.yaml.serializer.ScalarNode containing the included
-    filename, and an include directory path to search for matching files, load the given YAML
-    filename (ignoring the given loader so we can use our own) and return its contents as a data
-    structure of nested dicts and lists. If the filename is relative, probe for it within 1. the
-    current working directory and 2. the given include directory.
+    Given a ruamel.yaml.loader.Loader, a ruamel.yaml.nodes.ScalarNode containing the included
+    filename (or a list containing multiple such filenames), and an include directory path to search
+    for matching files, load the given YAML filenames (ignoring the given loader so we can use our
+    own) and return their contents as data structure of nested dicts, lists, etc. If the given
+    filename node's value is a scalar string, then the return value will be a single value. But if
+    the given node value is a list, then the return value will be a list of values, one per loaded
+    configuration file.
+
+    If a filename is relative, probe for it within 1. the current working directory and 2. the given
+    include directory.
 
     Raise FileNotFoundError if an included file was not found.
     '''
     include_directories = [os.getcwd(), os.path.abspath(include_directory)]
-    include_filename = os.path.expanduser(filename_node.value)
 
-    if not os.path.isabs(include_filename):
-        candidate_filenames = [
-            os.path.join(directory, include_filename) for directory in include_directories
+    if isinstance(filename_node.value, str):
+        return probe_and_include_file(filename_node.value, include_directories)
+
+    if (
+        isinstance(filename_node.value, list)
+        and len(filename_node.value)
+        and isinstance(filename_node.value[0], ruamel.yaml.nodes.ScalarNode)
+    ):
+        # Reversing the values ensures the correct ordering if these includes are subsequently
+        # merged together.
+        return [
+            probe_and_include_file(node.value, include_directories)
+            for node in reversed(filename_node.value)
         ]
 
-        for candidate_filename in candidate_filenames:
-            if os.path.exists(candidate_filename):
-                include_filename = candidate_filename
-                break
-        else:
-            raise FileNotFoundError(
-                f'Could not find include {filename_node.value} at {" or ".join(candidate_filenames)}'
-            )
-
-    return load_configuration(include_filename)
+    raise ValueError(
+        '!include value is not supported; use a single filename or a list of filenames'
+    )
 
 
 def raise_retain_node_error(loader, node):
@@ -53,7 +88,7 @@ def raise_retain_node_error(loader, node):
             'The !retain tag may only be used within a configuration file containing a merged !include tag.'
         )
 
-    raise ValueError('The !retain tag may only be used on a YAML mapping or sequence.')
+    raise ValueError('The !retain tag may only be used on a mapping or list.')
 
 
 def raise_omit_node_error(loader, node):
@@ -65,14 +100,14 @@ def raise_omit_node_error(loader, node):
     tags are handled by deep_merge_nodes() below.
     '''
     raise ValueError(
-        'The !omit tag may only be used on a scalar (e.g., string) list element within a configuration file containing a merged !include tag.'
+        'The !omit tag may only be used on a scalar (e.g., string) or list element within a configuration file containing a merged !include tag.'
     )
 
 
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
-    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    separate YAML configuration files. Example syntax: `option: !include common.yaml`
     '''
 
     def __init__(self, preserve_quotes=None, loader=None, include_directory=None):
@@ -81,6 +116,9 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
             '!include',
             functools.partial(include_configuration, include_directory=include_directory),
         )
+
+        # These are catch-all error handlers for tags that don't get applied and removed by
+        # deep_merge_nodes() below.
         self.add_constructor('!retain', raise_retain_node_error)
         self.add_constructor('!omit', raise_omit_node_error)
 
@@ -90,8 +128,8 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
         using the YAML '<<' merge key. Example syntax:
 
         ```
-        retention:
-            keep_daily: 1
+        option:
+            sub_option: 1
 
         <<: !include common.yaml
         ```
@@ -104,9 +142,15 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
 
         for index, (key_node, value_node) in enumerate(node.value):
             if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
-                included_value = representer.represent_data(self.construct_object(value_node))
-                node.value[index] = (key_node, included_value)
+                # Replace the merge include with a sequence of included configuration nodes ready
+                # for merging. The construct_object() call here triggers include_configuration()
+                # among other constructors.
+                node.value[index] = (
+                    key_node,
+                    representer.represent_data(self.construct_object(value_node)),
+                )
 
+        # This super().flatten_mapping() call actually performs "<<" merges.
         super(Include_constructor, self).flatten_mapping(node)
 
         node.value = deep_merge_nodes(node.value)
@@ -138,7 +182,12 @@ def load_configuration(filename):
         file_contents = file.read()
         config = yaml.load(file_contents)
 
-        if config and 'constants' in config:
+        try:
+            has_constants = bool(config and 'constants' in config)
+        except TypeError:
+            has_constants = False
+
+        if has_constants:
             for key, value in config['constants'].items():
                 value = json.dumps(value)
                 file_contents = file_contents.replace(f'{{{key}}}', value.strip('"'))
@@ -149,53 +198,92 @@ def load_configuration(filename):
         return config
 
 
-def filter_omitted_nodes(nodes):
+def filter_omitted_nodes(nodes, values):
     '''
-    Given a list of nodes, return a filtered list omitting any nodes with an "!omit" tag or with a
-    value matching such nodes.
+    Given a nested borgmatic configuration data structure as a list of tuples in the form of:
+
+    [
+        (
+            ruamel.yaml.nodes.ScalarNode as a key,
+            ruamel.yaml.nodes.MappingNode or other Node as a value,
+        ),
+        ...
+    ]
+
+    ... and a combined list of all values for those nodes, return a filtered list of the values,
+    omitting any that have an "!omit" tag (or with a value matching such nodes).
+
+    But if only a single node is given, bail and return the given values unfiltered, as "!omit" only
+    applies when there are merge includes (and therefore multiple nodes).
     '''
-    omitted_values = tuple(node.value for node in nodes if node.tag == '!omit')
+    if len(nodes) <= 1:
+        return values
 
-    return [node for node in nodes if node.value not in omitted_values]
+    omitted_values = tuple(node.value for node in values if node.tag == '!omit')
+
+    return [node for node in values if node.value not in omitted_values]
 
 
-DELETED_NODE = object()
+def merge_values(nodes):
+    '''
+    Given a nested borgmatic configuration data structure as a list of tuples in the form of:
+
+    [
+        (
+            ruamel.yaml.nodes.ScalarNode as a key,
+            ruamel.yaml.nodes.MappingNode or other Node as a value,
+        ),
+        ...
+    ]
+
+    ... merge its sequence or mapping node values and return the result. For sequence nodes, this
+    means appending together its contained lists. For mapping nodes, it means merging its contained
+    dicts.
+    '''
+    return functools.reduce(operator.add, (value.value for key, value in nodes))
 
 
 def deep_merge_nodes(nodes):
     '''
     Given a nested borgmatic configuration data structure as a list of tuples in the form of:
 
+    [
         (
             ruamel.yaml.nodes.ScalarNode as a key,
             ruamel.yaml.nodes.MappingNode or other Node as a value,
         ),
+        ...
+    ]
 
-    ... deep merge any node values corresponding to duplicate keys and return the result. If
-    there are colliding keys with non-MappingNode values (e.g., integers or strings), the last
-    of the values wins.
+    ... deep merge any node values corresponding to duplicate keys and return the result. The
+    purpose of merging like this is to support, for instance, merging one borgmatic configuration
+    file into another for reuse, such that a configuration option with sub-options does not
+    completely replace the corresponding option in a merged file.
+
+    If there are colliding keys with scalar values (e.g., integers or strings), the last of the
+    values wins.
 
     For instance, given node values of:
 
         [
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option1'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='1')
                     ),
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='7')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='2')
                     ),
                 ]),
             ),
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
                         ScalarNode(tag='tag:yaml.org,2002:int', value='5')
                     ),
                 ]),
@@ -206,88 +294,95 @@ def deep_merge_nodes(nodes):
 
         [
             (
-                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
                 MappingNode(tag='tag:yaml.org,2002:map', value=[
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
-                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option1'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='1')
                     ),
                     (
-                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='sub_option2'),
                         ScalarNode(tag='tag:yaml.org,2002:int', value='5')
                     ),
                 ]),
             ),
         ]
 
+    This function supports multi-way merging, meaning that if the same option name exists three or
+    more times (at the same scope level), all of those instances get merged together.
+
     If a mapping or sequence node has a YAML "!retain" tag, then that node is not merged.
 
-    The purpose of deep merging like this is to support, for instance, merging one borgmatic
-    configuration file into another for reuse, such that a configuration option with sub-options
-    does not completely replace the corresponding option in a merged file.
-
-    Raise ValueError if a merge is implied using two incompatible types.
+    Raise ValueError if a merge is implied using multiple incompatible types.
     '''
-    # Map from original node key/value to the replacement merged node. DELETED_NODE as a replacement
-    # node indications deletion.
-    replaced_nodes = {}
+    merged_nodes = []
 
-    # To find nodes that require merging, compare each node with each other node.
-    for a_key, a_value in nodes:
-        for b_key, b_value in nodes:
-            # If we've already considered one of the nodes for merging, skip it.
-            if (a_key, a_value) in replaced_nodes or (b_key, b_value) in replaced_nodes:
-                continue
+    def get_node_key_name(node):
+        return node[0].value
 
-            # If the keys match and the values are different, we need to merge these two A and B nodes.
-            if a_key.tag == b_key.tag and a_key.value == b_key.value and a_value != b_value:
-                if not type(a_value) is type(b_value):
-                    raise ValueError(
-                        f'Incompatible types found when trying to merge "{a_key.value}:" values across configuration files: {type(a_value).id} and {type(b_value).id}'
+    # Bucket the nodes by their keys. Then merge all of the values sharing the same key.
+    for key_name, grouped_nodes in itertools.groupby(
+        sorted(nodes, key=get_node_key_name), get_node_key_name
+    ):
+        grouped_nodes = list(grouped_nodes)
+
+        # The merged node inherits its attributes from the final node in the group.
+        (last_node_key, last_node_value) = grouped_nodes[-1]
+        value_types = set(type(value) for (_, value) in grouped_nodes)
+
+        if len(value_types) > 1:
+            raise ValueError(
+                f'Incompatible types found when trying to merge "{key_name}:" values across configuration files: {", ".join(value_type.id for value_type in value_types)}'
+            )
+
+        # If we're dealing with MappingNodes, recurse and merge its values as well.
+        if ruamel.yaml.nodes.MappingNode in value_types:
+            # A "!retain" tag says to skip deep merging for this node. Replace the tag so
+            # downstream schema validation doesn't break on our application-specific tag.
+            if last_node_value.tag == '!retain' and len(grouped_nodes) > 1:
+                last_node_value.tag = 'tag:yaml.org,2002:map'
+                merged_nodes.append((last_node_key, last_node_value))
+            else:
+                merged_nodes.append(
+                    (
+                        last_node_key,
+                        ruamel.yaml.nodes.MappingNode(
+                            tag=last_node_value.tag,
+                            value=deep_merge_nodes(merge_values(grouped_nodes)),
+                            start_mark=last_node_value.start_mark,
+                            end_mark=last_node_value.end_mark,
+                            flow_style=last_node_value.flow_style,
+                            comment=last_node_value.comment,
+                            anchor=last_node_value.anchor,
+                        ),
                     )
+                )
 
-                # Since we're merging into the B node, consider the A node a duplicate and remove it.
-                replaced_nodes[(a_key, a_value)] = DELETED_NODE
+            continue
 
-                # If we're dealing with MappingNodes, recurse and merge its values as well.
-                if isinstance(b_value, ruamel.yaml.nodes.MappingNode):
-                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
-                    # downstream schema validation doesn't break on our application-specific tag.
-                    if b_value.tag == '!retain':
-                        b_value.tag = 'tag:yaml.org,2002:map'
-                    else:
-                        replaced_nodes[(b_key, b_value)] = (
-                            b_key,
-                            ruamel.yaml.nodes.MappingNode(
-                                tag=b_value.tag,
-                                value=deep_merge_nodes(a_value.value + b_value.value),
-                                start_mark=b_value.start_mark,
-                                end_mark=b_value.end_mark,
-                                flow_style=b_value.flow_style,
-                                comment=b_value.comment,
-                                anchor=b_value.anchor,
-                            ),
-                        )
-                # If we're dealing with SequenceNodes, merge by appending one sequence to the other.
-                elif isinstance(b_value, ruamel.yaml.nodes.SequenceNode):
-                    # A "!retain" tag says to skip deep merging for this node. Replace the tag so
-                    # downstream schema validation doesn't break on our application-specific tag.
-                    if b_value.tag == '!retain':
-                        b_value.tag = 'tag:yaml.org,2002:seq'
-                    else:
-                        replaced_nodes[(b_key, b_value)] = (
-                            b_key,
-                            ruamel.yaml.nodes.SequenceNode(
-                                tag=b_value.tag,
-                                value=filter_omitted_nodes(a_value.value + b_value.value),
-                                start_mark=b_value.start_mark,
-                                end_mark=b_value.end_mark,
-                                flow_style=b_value.flow_style,
-                                comment=b_value.comment,
-                                anchor=b_value.anchor,
-                            ),
-                        )
+        # If we're dealing with SequenceNodes, merge by appending sequences together.
+        if ruamel.yaml.nodes.SequenceNode in value_types:
+            if last_node_value.tag == '!retain' and len(grouped_nodes) > 1:
+                last_node_value.tag = 'tag:yaml.org,2002:seq'
+                merged_nodes.append((last_node_key, last_node_value))
+            else:
+                merged_nodes.append(
+                    (
+                        last_node_key,
+                        ruamel.yaml.nodes.SequenceNode(
+                            tag=last_node_value.tag,
+                            value=filter_omitted_nodes(grouped_nodes, merge_values(grouped_nodes)),
+                            start_mark=last_node_value.start_mark,
+                            end_mark=last_node_value.end_mark,
+                            flow_style=last_node_value.flow_style,
+                            comment=last_node_value.comment,
+                            anchor=last_node_value.anchor,
+                        ),
+                    )
+                )
 
-    return [
-        replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE
-    ]
+            continue
+
+        merged_nodes.append((last_node_key, last_node_value))
+
+    return merged_nodes

borgmatic/config/normalize.py

@@ -10,7 +10,11 @@ def normalize_sections(config_filename, config):
 
     Raise ValueError if the "prefix" option is set in both "location" and "consistency" sections.
     '''
-    location = config.get('location') or {}
+    try:
+        location = config.get('location') or {}
+    except AttributeError:
+        raise ValueError('Configuration does not contain any options')
+
     storage = config.get('storage') or {}
     consistency = config.get('consistency') or {}
     hooks = config.get('hooks') or {}

borgmatic/config/schema.yaml

@@ -210,6 +210,13 @@ properties:
             "borgmatic restore" from finding any database dumps created before
             the change. Defaults to ~/.borgmatic
         example: /tmp/borgmatic
+    store_config_files:
+        type: boolean
+        description: |
+            Store configuration files used to create a backup in the backup
+            itself. Defaults to true. Changing this to false prevents "borgmatic
+            bootstrap" from extracting configuration files from the backup. 
+        example: true
     source_directories_must_exist:
         type: boolean
         description: |
@@ -365,19 +372,19 @@ properties:
         type: boolean
         description: |
             Bypass Borg error about a repository that has been moved. Defaults
-            to false.
+            to not bypassing.
         example: true
     unknown_unencrypted_repo_access_is_ok:
         type: boolean
         description: |
             Bypass Borg error about a previously unknown unencrypted repository.
-            Defaults to false.
+            Defaults to not bypassing.
         example: true
     check_i_know_what_i_am_doing:
         type: boolean
         description: |
-            Bypass Borg confirmation about check with repair option.
-            Defaults to false.
+            Bypass Borg confirmation about check with repair option. Defaults to
+            an interactive prompt from Borg.
         example: true
     extra_borg_options:
         type: object

borgmatic/hooks/ntfy.py

@@ -28,8 +28,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
         state_config = hook_config.get(
             state.name.lower(),
             {
-                'title': f'A Borgmatic {state.name} event happened',
-                'message': f'A Borgmatic {state.name} event happened',
+                'title': f'A borgmatic {state.name} event happened',
+                'message': f'A borgmatic {state.name} event happened',
                 'priority': 'default',
                 'tags': 'borgmatic',
             },

docs/SECURITY.md

@@ -16,4 +16,4 @@ each.
 If you find a security vulnerability, please [file a
 ticket](https://torsion.org/borgmatic/#issues) or [send email
 directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
-back within a few days at most, and generally sooner.
+back within a few days at most and generally sooner.

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -17,7 +17,7 @@ feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
 instead.
 
 You can specify `before_backup` hooks to perform preparation steps before
-running backups, and specify `after_backup` hooks to perform cleanup steps
+running backups and specify `after_backup` hooks to perform cleanup steps
 afterwards. Here's an example:
 
 ```yaml

docs/how-to/backup-your-databases.md

@@ -338,6 +338,28 @@ within the database dump:
 borgmatic restore --archive latest --database users --schema tentant1
 ```
 
+### Restore to an alternate host
+<span class="minilink minilink-addedin">New in version 1.7.15</span>
+A database dump can be restored to a host other than the one from which it was
+originally dumped. The connection parameters like the username, password, and
+port can also be changed. This can be done from the command line:
+
+```bash
+borgmatic restore --archive latest --database users --hostname database2.example.org --port 5433 --username postgres --password trustsome1
+```
+
+Or from the configuration file:
+
+```yaml
+postgresql_databases:
+    - name: users
+        hostname: database1.example.org
+        restore_hostname: database1.example.org
+        restore_port: 5433
+        restore_username: postgres
+        restore_password: trustsome1
+```
+
 
 ### Limitations
 

docs/how-to/deal-with-very-large-backups.md

@@ -189,7 +189,7 @@ check_repositories:
     - path/of/repository_to_check.borg
 ```
 
-Finally, you can override your configuration file's consistency checks, and
+Finally, you can override your configuration file's consistency checks and
 run particular checks via the command-line. For instance:
 
 ```bash

docs/how-to/develop-on-borgmatic.md

@@ -149,8 +149,8 @@ See the Black, Flake8, and isort documentation for more information.
 
 Each pull request triggers a continuous integration build which runs the test
 suite. You can view these builds on
-[build.torsion.org](https://build.torsion.org/borgmatic-collective/borgmatic), and they're
-also linked from the commits list on each pull request.
+[build.torsion.org](https://build.torsion.org/borgmatic-collective/borgmatic),
+and they're also linked from the commits list on each pull request.
 
 ## Documentation development
 

docs/how-to/extract-a-backup.md

@@ -44,7 +44,6 @@ entire contents of the archive to the current directory, so make sure you're
 in the right place before running the command—or see below about the
 `--destination` flag.
 
-
 ## Repository selection
 
 If you have a single repository in your borgmatic configuration file(s), no
@@ -145,3 +144,29 @@ When you're all done exploring your files, unmount your mount point. No
 ```bash
 borgmatic umount --mount-point /mnt
 ```
+
+## Extract the configuration files used to create an archive
+
+<span class="minilink minilink-addedin">New in version 1.7.15</span> borgmatic
+automatically stores all the configuration files used to create an archive inside the
+archive itself. This is useful in cases where you've lost a configuration
+file or you want to see what configurations were used to create a particular
+archive.
+
+To extract the configuration files from an archive, use the `config bootstrap` action. For example:
+
+```bash 
+borgmatic config bootstrap --repository repo.borg --destination /tmp
+```
+
+This extracts the configuration file from the latest archive in the repository `repo.borg` to `/tmp/etc/borgmatic/config.yaml`, assuming that the only configuration file used to create this archive was located at `/etc/borgmatic/config.yaml` when the archive was created.
+
+Note that to run the `config bootstrap` action, you don't need to have a borgmatic configuration file. You only need to specify the repository to use via the `--repository` flag; borgmatic will figure out the rest.
+
+If a destination directory is not specified, the configuration files will be extracted to their original locations, silently **overwriting** any configuration files that may already exist. For example, if a configuration file was located at `/etc/borgmatic/config.yaml` when the archive was created, it will be extracted to `/etc/borgmatic/config.yaml` too.
+
+If you want to extract the configuration file from a specific archive, use the `--archive` flag:
+
+```bash
+borgmatic config bootstrap --repository repo.borg --archive host-2023-01-02T04:06:07.080910 --destination /tmp
+```

docs/how-to/make-per-application-backups.md

@@ -153,7 +153,7 @@ in newer versions of borgmatic.
 Once you have multiple different configuration files, you might want to share
 common configuration options across these files with having to copy and paste
 them. To achieve this, you can put fragments of common configuration options
-into a file, and then include or inline that file into one or more borgmatic
+into a file and then include or inline that file into one or more borgmatic
 configuration files.
 
 Let's say that you want to include common consistency check configuration across all
@@ -233,18 +233,57 @@ checks:
 <span class="minilink minilink-addedin">Prior to version 1.8.0</span> These
 options were organized into sections like `retention:` and `consistency:`.
 
-Once this include gets merged in, the resulting configuration would have all
-of the options from the original configuration file *and* the options from the
+Once this include gets merged in, the resulting configuration has all of the
+options from the original configuration file *and* the options from the
 include.
 
 Note that this `<<` include merging syntax is only for merging in mappings
-(configuration options and their values). But if you'd like to include a
-single value directly, please see the above about standard includes.
+(configuration options and their values). If you'd like to include a single
+value directly, please see above about standard includes.
 
-Additionally, there is a limitation preventing multiple `<<` include merges
-per file or option value. So for instance, that means you can do one `<<`
-merge at the global level, another `<<` within each nested option value, etc.
-(This is a YAML limitation.)
+
+### Multiple merge includes
+
+borgmatic has a limitation preventing multiple `<<` include merges per file or
+option value. This means you can do a single `<<` merge at the global level,
+another `<<` within each nested option value, etc. (This is a YAML
+limitation.) For instance:
+
+```yaml
+repositories:
+   - path: repo.borg
+
+# This won't work! You can't do multiple merges like this at the same level.
+<<: !include common1.yaml
+<<: !include common2.yaml
+```
+
+But read on for a way around this.
+
+<span class="minilink minilink-addedin">New in version 1.8.1</span> You can
+include and merge multiple configuration files all at once. For instance:
+
+```yaml
+repositories:
+   - path: repo.borg
+
+<<: !include [common1.yaml, common2.yaml, common3.yaml]
+```
+
+This merges in each included configuration file in turn, such that later files
+replace the options in earlier ones.
+
+Here's another way to do the same thing:
+
+```yaml
+repositories:
+   - path: repo.borg
+
+<<: !include
+    - common1.yaml
+    - common2.yaml
+    - common3.yaml
+```
 
 
 ### Deep merge
@@ -309,8 +348,8 @@ source_directories:
     - /etc
 ```
 
-<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
-this option in the `location:` section of your configuration.
+<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put the
+`source_directories` option in the `location:` section of your configuration.
 
 Once this include gets merged in, the resulting configuration will have a
 `source_directories` value of `/etc` and `/var`—with `/home` omitted.
@@ -442,8 +481,8 @@ command-line via the `--override` flag. Here's an example:
 borgmatic create --override remote_path=/usr/local/bin/borg1
 ```
 
-What this does is load your configuration files, and for each one, disregard
-the configured value for the `remote_path` option, and use the value of
+What this does is load your configuration files and for each one, disregard
+the configured value for the `remote_path` option and use the value of
 `/usr/local/bin/borg1` instead.
 
 You can even override nested values or multiple values at once. For instance:

docs/how-to/monitor-your-backups.md

@@ -106,7 +106,7 @@ on_error:
 ```
 
 In this example, when the error occurs, borgmatic interpolates runtime values
-into the hook command: the borgmatic configuration filename, and the path of
+into the hook command: the borgmatic configuration filename and the path of
 the repository. Here's the full set of supported variables you can use here:
 
  * `configuration_filename`: borgmatic configuration filename in which the
@@ -118,7 +118,7 @@ the repository. Here's the full set of supported variables you can use here:
    occurred without running a command)
 
 Note that borgmatic runs the `on_error` hooks only for `create`, `prune`,
-`compact`, or `check` actions or hooks in which an error occurs, and not other
+`compact`, or `check` actions/hooks in which an error occurs and not other
 actions. borgmatic does not run `on_error` hooks if an error occurs within a
 `before_everything` or `after_everything` hook. For more about hooks, see the
 [borgmatic hooks
@@ -150,7 +150,7 @@ hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
 the `create`, `prune`, `compact`, or `check` actions are run.
 
 Then, if the actions complete successfully, borgmatic notifies Healthchecks of
-the success after the `after_backup` hooks run, and includes borgmatic logs in
+the success after the `after_backup` hooks run and includes borgmatic logs in
 the payload data sent to Healthchecks. This means that borgmatic logs show up
 in the Healthchecks UI, although be aware that Healthchecks currently has a
 10-kilobyte limit for the logs in each ping.
@@ -304,17 +304,17 @@ ntfy:
     topic: my-unique-topic
     server: https://ntfy.my-domain.com
     start:
-        title: A Borgmatic backup started
+        title: A borgmatic backup started
         message: Watch this space...
         tags: borgmatic
         priority: min
     finish:
-        title: A Borgmatic backup completed successfully
+        title: A borgmatic backup completed successfully
         message: Nice!
         tags: borgmatic,+1
         priority: min
     fail:
-        title: A Borgmatic backup failed
+        title: A borgmatic backup failed
         message: You should probably fix it
         tags: borgmatic,-1,skull
         priority: max
@@ -336,7 +336,7 @@ output formatted as JSON.
 
 Note that when you specify the `--json` flag, Borg's other non-JSON output is
 suppressed so as not to interfere with the captured JSON. Also note that JSON
-output only shows up at the console, and not in syslog.
+output only shows up at the console and not in syslog.
 
 
 ### Latest backups

docs/how-to/set-up-backups.md

@@ -59,11 +59,11 @@ borgmatic's path.
 
 ### Global install option
 
-If you try the user site installation above, and have problems making
-borgmatic commands runnable on your system `PATH`, an alternate approach is to
-install borgmatic globally.
+If you try the user site installation above and have problems making borgmatic
+commands runnable on your system `PATH`, an alternate approach is to install
+borgmatic globally.
 
-The following uninstalls borgmatic, and then reinstalls it such that borgmatic
+The following uninstalls borgmatic and then reinstalls it such that borgmatic
 commands are on the default system `PATH`:
 
 ```bash
@@ -229,7 +229,7 @@ sudo borgmatic rcreate --encryption repokey-aes-ocb
 certain platforms like ARM64.)
 
 This uses the borgmatic configuration file you created above to determine
-which local or remote repository to create, and encrypts it with the
+which local or remote repository to create and encrypts it with the
 encryption passphrase specified there if one is provided. Read about [Borg
 encryption
 modes](https://borgbackup.readthedocs.io/en/stable/usage/init.html#encryption-mode-tldr)

docs/how-to/upgrade.md

@@ -63,7 +63,7 @@ and, if desired, replace your original configuration file with it.
 ### Upgrading from borgmatic 1.0.x
 
 borgmatic changed its configuration file format in version 1.1.0 from
-INI-style to YAML. This better supports validation, and has a more natural way
+INI-style to YAML. This better supports validation and has a more natural way
 to express lists of values. To upgrade your existing configuration, first
 upgrade to the last version of borgmatic to support converting configuration:
 borgmatic 1.7.14.

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.8.0'
+VERSION = '1.8.1'
 
 
 setup(

tests/integration/config/test_load.py

@@ -44,6 +44,14 @@ def test_load_configuration_replaces_complex_constants():
     assert module.load_configuration('config.yaml') == {'key': {'subkey': 'value'}}
 
 
+def test_load_configuration_with_only_integer_value_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    config_file = io.StringIO('33')
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+    assert module.load_configuration('config.yaml') == 33
+
+
 def test_load_configuration_inlines_include_relative_to_current_directory():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -124,6 +132,37 @@ def test_load_configuration_raises_if_absolute_include_does_not_exist():
         assert module.load_configuration('config.yaml')
 
 
+def test_load_configuration_inlines_multiple_file_include_as_list():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(True)
+    flexmock(module.os.path).should_receive('exists').never()
+    include1_file = io.StringIO('value1')
+    include1_file.name = '/root/include1.yaml'
+    builtins.should_receive('open').with_args('/root/include1.yaml').and_return(include1_file)
+    include2_file = io.StringIO('value2')
+    include2_file.name = '/root/include2.yaml'
+    builtins.should_receive('open').with_args('/root/include2.yaml').and_return(include2_file)
+    config_file = io.StringIO('key: !include [/root/include1.yaml, /root/include2.yaml]')
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {'key': ['value2', 'value1']}
+
+
+def test_load_configuration_include_with_unsupported_filename_type_raises():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(True)
+    flexmock(module.os.path).should_receive('exists').never()
+    config_file = io.StringIO('key: !include {path: /root/include.yaml}')
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    with pytest.raises(ValueError):
+        module.load_configuration('config.yaml')
+
+
 def test_load_configuration_merges_include():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -149,6 +188,43 @@ def test_load_configuration_merges_include():
     assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
 
 
+def test_load_configuration_merges_multiple_file_include():
+    builtins = flexmock(sys.modules['builtins'])
+    flexmock(module.os).should_receive('getcwd').and_return('/tmp')
+    flexmock(module.os.path).should_receive('isabs').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    include1_file = io.StringIO(
+        '''
+        foo: bar
+        baz: quux
+        original: yes
+        '''
+    )
+    include1_file.name = 'include1.yaml'
+    builtins.should_receive('open').with_args('/tmp/include1.yaml').and_return(include1_file)
+    include2_file = io.StringIO(
+        '''
+        baz: second
+        '''
+    )
+    include2_file.name = 'include2.yaml'
+    builtins.should_receive('open').with_args('/tmp/include2.yaml').and_return(include2_file)
+    config_file = io.StringIO(
+        '''
+        foo: override
+        <<: !include [include1.yaml, include2.yaml]
+        '''
+    )
+    config_file.name = 'config.yaml'
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_file)
+
+    assert module.load_configuration('config.yaml') == {
+        'foo': 'override',
+        'baz': 'second',
+        'original': 'yes',
+    }
+
+
 def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values():
     builtins = flexmock(sys.modules['builtins'])
     flexmock(module.os).should_receive('getcwd').and_return('/tmp')
@@ -438,8 +514,9 @@ def test_raise_omit_node_error_raises():
         module.raise_omit_node_error(loader=flexmock(), node=flexmock())
 
 
-def test_filter_omitted_nodes():
-    nodes = [
+def test_filter_omitted_nodes_discards_values_with_omit_tag_and_also_equal_values():
+    nodes = [flexmock(), flexmock()]
+    values = [
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
@@ -448,11 +525,139 @@ def test_filter_omitted_nodes():
         module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
     ]
 
-    result = module.filter_omitted_nodes(nodes)
+    result = module.filter_omitted_nodes(nodes, values)
 
     assert [item.value for item in result] == ['a', 'c', 'a', 'c']
 
 
+def test_filter_omitted_nodes_keeps_all_values_when_given_only_one_node():
+    nodes = [flexmock()]
+    values = [
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='a'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='!omit', value='b'),
+        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='c'),
+    ]
+
+    result = module.filter_omitted_nodes(nodes, values)
+
+    assert [item.value for item in result] == ['a', 'b', 'c', 'a', 'b', 'c']
+
+
+def test_merge_values_combines_mapping_values():
+    nodes = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_hourly'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='24'
+                        ),
+                    ),
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='25'
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_nanosecondly'
+                        ),
+                        module.ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:int', value='1000'
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    values = module.merge_values(nodes)
+
+    assert len(values) == 4
+    assert values[0][0].value == 'keep_hourly'
+    assert values[0][1].value == '24'
+    assert values[1][0].value == 'keep_daily'
+    assert values[1][1].value == '7'
+    assert values[2][0].value == 'keep_daily'
+    assert values[2][1].value == '25'
+    assert values[3][0].value == 'keep_nanosecondly'
+    assert values[3][1].value == '1000'
+
+
+def test_merge_values_combines_sequence_values():
+    nodes = [
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.SequenceNode(
+                tag='tag:yaml.org,2002:seq',
+                value=[
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='1'),
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='2'),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.SequenceNode(
+                tag='tag:yaml.org,2002:seq',
+                value=[
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='3'),
+                ],
+            ),
+        ),
+        (
+            module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='option'),
+            module.ruamel.yaml.nodes.SequenceNode(
+                tag='tag:yaml.org,2002:seq',
+                value=[
+                    module.ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='4'),
+                ],
+            ),
+        ),
+    ]
+
+    values = module.merge_values(nodes)
+
+    assert len(values) == 4
+    assert values[0].value == '1'
+    assert values[1].value == '2'
+    assert values[2].value == '3'
+    assert values[3].value == '4'
+
+
 def test_deep_merge_nodes_replaces_colliding_scalar_values():
     node_values = [
         (
@@ -499,10 +704,10 @@ def test_deep_merge_nodes_replaces_colliding_scalar_values():
     assert section_key.value == 'retention'
     options = section_value.value
     assert len(options) == 2
-    assert options[0][0].value == 'keep_hourly'
-    assert options[0][1].value == '24'
-    assert options[1][0].value == 'keep_daily'
-    assert options[1][1].value == '5'
+    assert options[0][0].value == 'keep_daily'
+    assert options[0][1].value == '5'
+    assert options[1][0].value == 'keep_hourly'
+    assert options[1][1].value == '24'
 
 
 def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
@@ -553,10 +758,10 @@ def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
     assert section_key.value == 'retention'
     options = section_value.value
     assert len(options) == 3
-    assert options[0][0].value == 'keep_hourly'
-    assert options[0][1].value == '24'
-    assert options[1][0].value == 'keep_daily'
-    assert options[1][1].value == '7'
+    assert options[0][0].value == 'keep_daily'
+    assert options[0][1].value == '7'
+    assert options[1][0].value == 'keep_hourly'
+    assert options[1][1].value == '24'
     assert options[2][0].value == 'keep_minutely'
     assert options[2][1].value == '10'
 
@@ -629,10 +834,10 @@ def test_deep_merge_nodes_keeps_deeply_nested_values():
     assert section_key.value == 'storage'
     options = section_value.value
     assert len(options) == 2
-    assert options[0][0].value == 'lock_wait'
-    assert options[0][1].value == '5'
-    assert options[1][0].value == 'extra_borg_options'
-    nested_options = options[1][1].value
+    assert options[0][0].value == 'extra_borg_options'
+    assert options[1][0].value == 'lock_wait'
+    assert options[1][1].value == '5'
+    nested_options = options[0][1].value
     assert len(nested_options) == 2
     assert nested_options[0][0].value == 'init'
     assert nested_options[0][1].value == '--init-option'

tests/integration/config/test_validate.py

@@ -117,7 +117,7 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
     module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
 
 
-def test_parse_configuration_inlines_include():
+def test_parse_configuration_inlines_include_inside_deprecated_section():
     mock_config_and_schema(
         '''
         source_directories:

tests/unit/actions/test_create.py

@@ -40,6 +40,41 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
     )
 
 
+def test_run_create_with_store_config_files_false_does_not_create_borgmatic_manifest():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
+    flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
+    flexmock(module).should_receive('create_borgmatic_manifest').never()
+    flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({})
+    flexmock(module.borgmatic.hooks.dispatch).should_receive(
+        'call_hooks_even_if_unconfigured'
+    ).and_return({})
+    create_arguments = flexmock(
+        repository=None,
+        progress=flexmock(),
+        stats=flexmock(),
+        json=flexmock(),
+        list_files=flexmock(),
+    )
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
+
+    list(
+        module.run_create(
+            config_filename='test.yaml',
+            repository={'path': 'repo'},
+            config={'store_config_files': False},
+            hook_context={},
+            local_borg_version=None,
+            create_arguments=create_arguments,
+            global_arguments=global_arguments,
+            dry_run_label='',
+            local_path=None,
+            remote_path=None,
+        )
+    )
+
+
 def test_run_create_runs_with_selected_repository():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive(

tests/unit/borg/test_create.py

@@ -647,6 +647,53 @@ def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sourc
     )
 
 
+def test_create_archive_with_sources_and_used_config_paths_with_store_config_files_false_calls_borg_with_sources_and_no_config_paths():
+    flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
+    flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
+    flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('expand_directories').with_args(('foo', 'bar')).and_return(
+        ('foo', 'bar')
+    )
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('pattern_root_directories').and_return([])
+    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
+    flexmock(module).should_receive('expand_home_directories').and_return(())
+    flexmock(module).should_receive('write_pattern_file').and_return(None)
+    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module).should_receive('ensure_files_readable')
+    flexmock(module).should_receive('make_pattern_flags').and_return(())
+    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{DEFAULT_ARCHIVE_NAME}',)
+    )
+    environment = {'BORG_THINGY': 'YUP'}
+    flexmock(module.environment).should_receive('make_environment').and_return(environment)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=environment,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository_path='repo',
+        config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'store_config_files': False,
+        },
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']),
+    )
+
+
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
     flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER

tests/unit/borg/test_environment.py

@@ -19,28 +19,36 @@ def test_make_environment_with_ssh_command_should_set_environment():
     assert environment.get('BORG_RSH') == 'ssh -C'
 
 
-def test_make_environment_without_configuration_should_only_set_default_environment():
+def test_make_environment_without_configuration_should_not_set_environment():
     environment = module.make_environment({})
 
-    assert environment == {
-        'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
-        'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
-        'BORG_CHECK_I_KNOW_WHAT_I_AM_DOING': 'NO',
-    }
+    assert environment == {}
 
 
-def test_make_environment_with_relocated_repo_access_should_override_default():
+def test_make_environment_with_relocated_repo_access_true_should_set_environment_yes():
     environment = module.make_environment({'relocated_repo_access_is_ok': True})
 
     assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
 
 
-def test_make_environment_check_i_know_what_i_am_doing_should_override_default():
+def test_make_environment_with_relocated_repo_access_false_should_set_environment_no():
+    environment = module.make_environment({'relocated_repo_access_is_ok': False})
+
+    assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'no'
+
+
+def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environment_YES():
     environment = module.make_environment({'check_i_know_what_i_am_doing': True})
 
     assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'YES'
 
 
+def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environment_NO():
+    environment = module.make_environment({'check_i_know_what_i_am_doing': False})
+
+    assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'NO'
+
+
 def test_make_environment_with_integer_variable_value():
     environment = module.make_environment({'borg_files_cache_ttl': 40})
     assert environment.get('BORG_FILES_CACHE_TTL') == '40'

tests/unit/borg/test_flags.py

@@ -86,28 +86,28 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a
 
 
 @pytest.mark.parametrize(
-    'match_archives, archive_name_format,feature_available,expected_result',
+    'match_archives,archive_name_format,feature_available,expected_result',
     (
         (None, None, True, ()),
         (None, '', True, ()),
         (
             're:foo-.*',
-            '{hostname}-{now}',
+            '{hostname}-{now}',  # noqa: FS003
             True,
             ('--match-archives', 're:foo-.*'),
-        ),  # noqa: FS003
+        ),
         (
             'sh:foo-*',
-            '{hostname}-{now}',
+            '{hostname}-{now}',  # noqa: FS003
             False,
             ('--glob-archives', 'foo-*'),
-        ),  # noqa: FS003
+        ),
         (
             'foo-*',
-            '{hostname}-{now}',
+            '{hostname}-{now}',  # noqa: FS003
             False,
             ('--glob-archives', 'foo-*'),
-        ),  # noqa: FS003
+        ),
         (
             None,
             '{hostname}-docs-{now}',  # noqa: FS003
@@ -133,6 +133,18 @@ def test_make_repository_archive_flags_with_borg_features_joins_repository_and_a
             False,
             ('--glob-archives', '{hostname}-docs-*'),  # noqa: FS003
         ),
+        (
+            None,
+            '{now}',  # noqa: FS003
+            False,
+            (),
+        ),
+        (
+            None,
+            '{now}',  # noqa: FS003
+            True,
+            (),
+        ),
         (None, '{utcnow}-docs-{user}', False, ('--glob-archives', '*-docs-{user}')),  # noqa: FS003
     ),
 )

tests/unit/borg/test_prune.py

@@ -118,6 +118,19 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead():
     assert result == expected
 
 
+def test_make_prune_flags_ignores_keep_exclude_tags_in_config():
+    config = {
+        'keep_daily': 1,
+        'keep_exclude_tags': True,
+    }
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())
+
+    result = module.make_prune_flags(config, local_borg_version='1.2.3')
+
+    assert result == ('--keep-daily', '1')
+
+
 PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3')
 
 

tests/unit/config/test_load.py

@@ -0,0 +1,37 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_probe_and_include_file_with_absolute_path_skips_probing():
+    config = flexmock()
+    flexmock(module).should_receive('load_configuration').with_args('/etc/include.yaml').and_return(
+        config
+    ).once()
+
+    assert module.probe_and_include_file('/etc/include.yaml', ['/etc', '/var']) == config
+
+
+def test_probe_and_include_file_with_relative_path_probes_include_directories():
+    config = flexmock()
+    flexmock(module.os.path).should_receive('exists').with_args('/etc/include.yaml').and_return(
+        False
+    )
+    flexmock(module.os.path).should_receive('exists').with_args('/var/include.yaml').and_return(
+        True
+    )
+    flexmock(module).should_receive('load_configuration').with_args('/etc/include.yaml').never()
+    flexmock(module).should_receive('load_configuration').with_args('/var/include.yaml').and_return(
+        config
+    ).once()
+
+    assert module.probe_and_include_file('include.yaml', ['/etc', '/var']) == config
+
+
+def test_probe_and_include_file_with_relative_path_and_missing_files_raises():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module).should_receive('load_configuration').never()
+
+    with pytest.raises(FileNotFoundError):
+        module.probe_and_include_file('include.yaml', ['/etc', '/var'])

tests/unit/config/test_normalize.py

@@ -111,6 +111,13 @@ def test_normalize_sections_with_different_umask_values_raises():
         module.normalize_sections('test.yaml', config)
 
 
+def test_normalize_sections_with_only_scalar_raises():
+    config = 33
+
+    with pytest.raises(ValueError):
+        module.normalize_sections('test.yaml', config)
+
+
 @pytest.mark.parametrize(
     'config,expected_config,produces_logs',
     (

tests/unit/hooks/test_ntfy.py

@@ -10,8 +10,8 @@ custom_base_url = 'https://ntfy.example.com'
 topic = 'borgmatic-unit-testing'
 
 custom_message_config = {
-    'title': 'Borgmatic unit testing',
-    'message': 'Borgmatic unit testing',
+    'title': 'borgmatic unit testing',
+    'message': 'borgmatic unit testing',
     'priority': 'min',
     'tags': '+1',
 }
@@ -26,8 +26,8 @@ custom_message_headers = {
 
 def return_default_message_headers(state=Enum):
     headers = {
-        'X-Title': f'A Borgmatic {state.name} event happened',
-        'X-Message': f'A Borgmatic {state.name} event happened',
+        'X-Title': f'A borgmatic {state.name} event happened',
+        'X-Message': f'A borgmatic {state.name} event happened',
         'X-Priority': 'default',
         'X-Tags': 'borgmatic',
     }