Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1056
Reviewed-by: Dan Helfman <witten@torsion.org>

.gitea/workflows/build.yaml

@@ -17,6 +17,7 @@ jobs:
   docs:
     needs: [test]
     runs-on: host
+    if: gitea.event_name == 'push'
     env:
       IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
 

NEWS

@@ -1,3 +1,35 @@
+2.0.4
+ * #1072: Fix path rewriting for non-root patterns in the ZFS, Btrfs, and LVM hooks.
+ * #1073: Clarify the documentation about when an "after: error" command hook runs and how it
+   differs from other hooks:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #1075: Fix an incorrect warning about Borg placeholders being unsupported in a command hook.
+ * #1080: If the exact same "everything" command hook is present in multiple configuration files,
+   only run it once.
+
+2.0.3
+ * #1065: Fix a regression in monitoring hooks in which an error pinged the finish state instead of
+   the fail state.
+ * #1066: Add a "states" option to command hooks, so you can optionally skip an "after" hook if
+   borgmatic encounters an error.
+ * #1071: Fix an error in the LVM hook when removing a snapshot directory.
+
+2.0.2
+ * #1035: Document potential performance issues and workarounds with the ZFS, Btrfs, and LVM hooks:
+   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
+ * #1053: Display a nicer error message when the "recreate" action encounters an archive that
+   already exists.
+ * #1059: Fix a regression in which soft failure exit codes in command hooks were not respected.
+ * #1060: Fix action command hooks getting run too many times when multiple borgmatic actions are
+   executed (implicitly or explicitly).
+ * #1060: Don't run action command hooks for actions listed in the "skip_actions" option.
+ * #1062: Fix a regression that broke environment variable interpolation.
+ * #1063: List the configured "when" action names in the log entries for command hooks.
+
+2.0.1
+ * #1057: Fix argument parsing to avoid using Python 3.12+ string features. Now borgmatic will
+   work with Python 3.9, 3.10, and 3.11 again.
+
 2.0.0
  * TL;DR: More flexible, completely revamped command hooks. All configuration options settable on
    the command-line. New configuration options for many command-line flags (including verbosity!).

README.md

@@ -40,8 +40,10 @@ checks:
       frequency: 2 weeks
 
 # Custom preparation scripts to run.
-before_backup:
-    - prepare-for-backup.sh
+commands:
+    - before: action
+      when: [create]
+      run: [prepare-for-backup.sh]
 
 # Databases to dump and include in backups.
 postgresql_databases:

borgmatic/actions/check.py

@@ -8,7 +8,7 @@ import pathlib
 import random
 import shutil
 
-import borgmatic.actions.create
+import borgmatic.actions.pattern
 import borgmatic.borg.check
 import borgmatic.borg.create
 import borgmatic.borg.environment
@@ -373,8 +373,8 @@ def collect_spot_check_source_paths(
             dry_run=True,
             repository_path=repository['path'],
             config=dict(config, list_details=True),
-            patterns=borgmatic.actions.create.process_patterns(
-                borgmatic.actions.create.collect_patterns(config),
+            patterns=borgmatic.actions.pattern.process_patterns(
+                borgmatic.actions.pattern.collect_patterns(config),
                 working_directory,
             ),
             local_borg_version=local_borg_version,

borgmatic/actions/create.py

@@ -1,272 +1,15 @@
-import glob
-import itertools
 import logging
-import os
-import pathlib
 
 import borgmatic.actions.json
 import borgmatic.borg.create
-import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.config.validate
-import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
+from borgmatic.actions import pattern
 
 logger = logging.getLogger(__name__)
 
 
-def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
-    '''
-    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
-    return it.
-    '''
-    try:
-        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
-    except ValueError:
-        raise ValueError(f'Invalid pattern: {pattern_line}')
-
-    try:
-        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
-        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
-    except ValueError:
-        pattern_style = default_style
-        path = remainder
-
-    return borgmatic.borg.pattern.Pattern(
-        path,
-        borgmatic.borg.pattern.Pattern_type(pattern_type),
-        borgmatic.borg.pattern.Pattern_style(pattern_style),
-        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
-    )
-
-
-def collect_patterns(config):
-    '''
-    Given a configuration dict, produce a single sequence of patterns comprised of the configured
-    source directories, patterns, excludes, pattern files, and exclude files.
-
-    The idea is that Borg has all these different ways of specifying includes, excludes, source
-    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
-    ease of manipulation within borgmatic.
-    '''
-    try:
-        return (
-            tuple(
-                borgmatic.borg.pattern.Pattern(
-                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
-                )
-                for source_directory in config.get('source_directories', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip())
-                for pattern_line in config.get('patterns', ())
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for exclude_line in config.get('exclude_patterns', ())
-            )
-            + tuple(
-                parse_pattern(pattern_line.strip())
-                for filename in config.get('patterns_from', ())
-                for pattern_line in open(filename).readlines()
-                if not pattern_line.lstrip().startswith('#')
-                if pattern_line.strip()
-            )
-            + tuple(
-                parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
-                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                )
-                for filename in config.get('exclude_from', ())
-                for exclude_line in open(filename).readlines()
-                if not exclude_line.lstrip().startswith('#')
-                if exclude_line.strip()
-            )
-        )
-    except (FileNotFoundError, OSError) as error:
-        logger.debug(error)
-
-        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
-
-
-def expand_directory(directory, working_directory):
-    '''
-    Given a directory path, expand any tilde (representing a user's home directory) and any globs
-    therein. Return a list of one or more resulting paths.
-
-    Take into account the given working directory so that relative paths are supported.
-    '''
-    expanded_directory = os.path.expanduser(directory)
-
-    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
-    # only available in Python 3.10+.
-    normalized_directory = os.path.join(working_directory or '', expanded_directory)
-    glob_paths = glob.glob(normalized_directory)
-
-    if not glob_paths:
-        return [expanded_directory]
-
-    working_directory_prefix = os.path.join(working_directory or '', '')
-
-    return [
-        (
-            glob_path
-            # If these are equal, that means we didn't add any working directory prefix above.
-            if normalized_directory == expanded_directory
-            # Remove the working directory prefix that we added above in order to make glob() work.
-            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
-            # hack.
-            else glob_path.removeprefix(working_directory_prefix)
-        )
-        for glob_path in glob_paths
-    ]
-
-
-def expand_patterns(patterns, working_directory=None, skip_paths=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    expand tildes and globs in each root pattern and expand just tildes in each non-root pattern.
-    The idea is that non-root patterns may be regular expressions or other pattern styles containing
-    "*" that borgmatic should not expand as a shell glob.
-
-    Return all the resulting patterns as a tuple.
-
-    If a set of paths are given to skip, then don't expand any patterns matching them.
-    '''
-    if patterns is None:
-        return ()
-
-    return tuple(
-        itertools.chain.from_iterable(
-            (
-                (
-                    borgmatic.borg.pattern.Pattern(
-                        expanded_path,
-                        pattern.type,
-                        pattern.style,
-                        pattern.device,
-                        pattern.source,
-                    )
-                    for expanded_path in expand_directory(pattern.path, working_directory)
-                )
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and pattern.path not in (skip_paths or ())
-                else (
-                    borgmatic.borg.pattern.Pattern(
-                        os.path.expanduser(pattern.path),
-                        pattern.type,
-                        pattern.style,
-                        pattern.device,
-                        pattern.source,
-                    ),
-                )
-            )
-            for pattern in patterns
-        )
-    )
-
-
-def device_map_patterns(patterns, working_directory=None):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
-    determine the identifier for the device on which the pattern's path resides—or None if the path
-    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
-    device field populated. But if the device field is already set, don't bother setting it again.
-
-    This is handy for determining whether two different pattern paths are on the same filesystem
-    (have the same device identifier).
-    '''
-    return tuple(
-        borgmatic.borg.pattern.Pattern(
-            pattern.path,
-            pattern.type,
-            pattern.style,
-            device=pattern.device
-            or (
-                os.stat(full_path).st_dev
-                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-                and os.path.exists(full_path)
-                else None
-            ),
-            source=pattern.source,
-        )
-        for pattern in patterns
-        for full_path in (os.path.join(working_directory or '', pattern.path),)
-    )
-
-
-def deduplicate_patterns(patterns):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
-    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
-    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
-    modification.
-
-    The one exception to deduplication is two paths are on different filesystems (devices). In that
-    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
-    one_file_system option is true).
-
-    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
-    child patterns, because it will naturally spider the contents of the parent pattern's path. And
-    there are cases where Borg coming across the same file twice will result in duplicate reads and
-    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
-    Borg.
-    '''
-    deduplicated = {}  # Use just the keys as an ordered set.
-
-    for pattern in patterns:
-        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-            deduplicated[pattern] = True
-            continue
-
-        parents = pathlib.PurePath(pattern.path).parents
-
-        # If another directory in the given list is a parent of current directory (even n levels up)
-        # and both are on the same filesystem, then the current directory is a duplicate.
-        for other_pattern in patterns:
-            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
-                continue
-
-            if any(
-                pathlib.PurePath(other_pattern.path) == parent
-                and pattern.device is not None
-                and other_pattern.device == pattern.device
-                for parent in parents
-            ):
-                break
-        else:
-            deduplicated[pattern] = True
-
-    return tuple(deduplicated.keys())
-
-
-def process_patterns(patterns, working_directory, skip_expand_paths=None):
-    '''
-    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
-    "root" patterns, returning the resulting root and non-root patterns as a list.
-
-    If any paths are given to skip, don't expand them.
-    '''
-    skip_paths = set(skip_expand_paths or ())
-
-    return list(
-        deduplicate_patterns(
-            device_map_patterns(
-                expand_patterns(
-                    patterns,
-                    working_directory=working_directory,
-                    skip_paths=skip_paths,
-                )
-            )
-        )
-    )
-
-
 def run_create(
     config_filename,
     repository,
@@ -310,7 +53,7 @@ def run_create(
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )
-        patterns = process_patterns(collect_patterns(config), working_directory)
+        patterns = pattern.process_patterns(pattern.collect_patterns(config), working_directory)
         active_dumps = borgmatic.hooks.dispatch.call_hooks(
             'dump_data_sources',
             config,
@@ -324,7 +67,9 @@ def run_create(
         # Process the patterns again in case any data source hooks updated them. Without this step,
         # we could end up with duplicate paths that cause Borg to hang when it tries to read from
         # the same named pipe twice.
-        patterns = process_patterns(patterns, working_directory, skip_expand_paths=config_paths)
+        patterns = pattern.process_patterns(
+            patterns, working_directory, skip_expand_paths=config_paths
+        )
         stream_processes = [process for processes in active_dumps.values() for process in processes]
 
         json_output = borgmatic.borg.create.create_archive(

borgmatic/actions/pattern.py

@@ -0,0 +1,292 @@
+import glob
+import itertools
+import logging
+import os
+import pathlib
+
+import borgmatic.borg.pattern
+
+logger = logging.getLogger(__name__)
+
+
+def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
+    '''
+    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
+    return it.
+    '''
+    try:
+        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
+    except ValueError:
+        raise ValueError(f'Invalid pattern: {pattern_line}')
+
+    try:
+        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
+        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
+    except ValueError:
+        pattern_style = default_style
+        path = remainder
+
+    return borgmatic.borg.pattern.Pattern(
+        path,
+        borgmatic.borg.pattern.Pattern_type(pattern_type),
+        borgmatic.borg.pattern.Pattern_style(pattern_style),
+        source=borgmatic.borg.pattern.Pattern_source.CONFIG,
+    )
+
+
+def collect_patterns(config):
+    '''
+    Given a configuration dict, produce a single sequence of patterns comprised of the configured
+    source directories, patterns, excludes, pattern files, and exclude files.
+
+    The idea is that Borg has all these different ways of specifying includes, excludes, source
+    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
+    ease of manipulation within borgmatic.
+    '''
+    try:
+        return (
+            tuple(
+                borgmatic.borg.pattern.Pattern(
+                    source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
+                )
+                for source_directory in config.get('source_directories', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip())
+                for pattern_line in config.get('patterns', ())
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for exclude_line in config.get('exclude_patterns', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip())
+                for filename in config.get('patterns_from', ())
+                for pattern_line in open(filename).readlines()
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for filename in config.get('exclude_from', ())
+                for exclude_line in open(filename).readlines()
+                if not exclude_line.lstrip().startswith('#')
+                if exclude_line.strip()
+            )
+        )
+    except (FileNotFoundError, OSError) as error:
+        logger.debug(error)
+
+        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
+
+
+def expand_directory(directory, working_directory):
+    '''
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+
+    Take into account the given working directory so that relative paths are supported.
+    '''
+    expanded_directory = os.path.expanduser(directory)
+
+    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
+    # only available in Python 3.10+.
+    normalized_directory = os.path.join(working_directory or '', expanded_directory)
+    glob_paths = glob.glob(normalized_directory)
+
+    if not glob_paths:
+        return [expanded_directory]
+
+    working_directory_prefix = os.path.join(working_directory or '', '')
+
+    return [
+        (
+            glob_path
+            # If these are equal, that means we didn't add any working directory prefix above.
+            if normalized_directory == expanded_directory
+            # Remove the working directory prefix that we added above in order to make glob() work.
+            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
+            # hack.
+            else glob_path.removeprefix(working_directory_prefix)
+        )
+        for glob_path in glob_paths
+    ]
+
+
+def expand_patterns(patterns, working_directory=None, skip_paths=None):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    expand tildes and globs in each root pattern and expand just tildes in each non-root pattern.
+    The idea is that non-root patterns may be regular expressions or other pattern styles containing
+    "*" that borgmatic should not expand as a shell glob.
+
+    Return all the resulting patterns as a tuple.
+
+    If a set of paths are given to skip, then don't expand any patterns matching them.
+    '''
+    if patterns is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(
+            (
+                (
+                    borgmatic.borg.pattern.Pattern(
+                        expanded_path,
+                        pattern.type,
+                        pattern.style,
+                        pattern.device,
+                        pattern.source,
+                    )
+                    for expanded_path in expand_directory(pattern.path, working_directory)
+                )
+                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.path not in (skip_paths or ())
+                else (
+                    borgmatic.borg.pattern.Pattern(
+                        os.path.expanduser(pattern.path),
+                        pattern.type,
+                        pattern.style,
+                        pattern.device,
+                        pattern.source,
+                    ),
+                )
+            )
+            for pattern in patterns
+        )
+    )
+
+
+def get_existent_path_or_parent(path):
+    '''
+    Given a path, return it if it exists. Otherwise, return the longest parent directory of the path
+    that exists. Return None if none of these paths exist.
+
+    This is used below for finding an existent path prefix of pattern's path, which is necessary if
+    the path contain globs or other special characters that we don't want to try to interpret
+    (because we want to leave that responsibility to Borg).
+    '''
+    if path.startswith('/e2e/'):
+        return None
+
+    try:
+        return next(
+            candidate_path
+            for candidate_path in (path,)
+            + tuple(str(parent) for parent in pathlib.PurePath(path).parents)
+            if os.path.exists(candidate_path)
+        )
+    except StopIteration:
+        return None
+
+
+def device_map_patterns(patterns, working_directory=None):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    determine the identifier for the device on which the pattern's path resides—or None if the path
+    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
+    device field populated. But if the device field is already set, don't bother setting it again.
+
+    This is handy for determining whether two different pattern paths are on the same filesystem
+    (have the same device identifier).
+
+    This function only considers the start of a pattern's path—from the start of the path up until
+    there's a path component with a glob or other non-literal character. If there are no such
+    characters, the whole path is considered. The rationale is that it's not feasible for borgmatic
+    to interpret Borg's patterns to see which actual files (and therefore devices) they map to. So
+    for instance, a pattern with a path of "/var/log/*/data" would end up with its device set to the
+    device of "/var/log"—ignoring the "/*/data" part due to that glob.
+
+    The one exception is that if a regular expression pattern path starts with "^", that will get
+    stripped off for purposes of determining its device.
+    '''
+    return tuple(
+        borgmatic.borg.pattern.Pattern(
+            pattern.path,
+            pattern.type,
+            pattern.style,
+            device=pattern.device or (os.stat(existent_path).st_dev if existent_path else None),
+            source=pattern.source,
+        )
+        for pattern in patterns
+        for existent_path in (
+            get_existent_path_or_parent(
+                os.path.join(working_directory or '', pattern.path.lstrip('^'))
+            ),
+        )
+    )
+
+
+def deduplicate_patterns(patterns):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
+    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
+    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
+    modification.
+
+    The one exception to deduplication is two paths are on different filesystems (devices). In that
+    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
+    one_file_system option is true).
+
+    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
+    child patterns, because it will naturally spider the contents of the parent pattern's path. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = {}  # Use just the keys as an ordered set.
+
+    for pattern in patterns:
+        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+            deduplicated[pattern] = True
+            continue
+
+        parents = pathlib.PurePath(pattern.path).parents
+
+        # If another directory in the given list is a parent of current directory (even n levels up)
+        # and both are on the same filesystem, then the current directory is a duplicate.
+        for other_pattern in patterns:
+            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+                continue
+
+            if any(
+                pathlib.PurePath(other_pattern.path) == parent
+                and pattern.device is not None
+                and other_pattern.device == pattern.device
+                for parent in parents
+            ):
+                break
+        else:
+            deduplicated[pattern] = True
+
+    return tuple(deduplicated.keys())
+
+
+def process_patterns(patterns, working_directory, skip_expand_paths=None):
+    '''
+    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
+    "root" patterns, returning the resulting root and non-root patterns as a list.
+
+    If any paths are given to skip, don't expand them.
+    '''
+    skip_paths = set(skip_expand_paths or ())
+
+    return list(
+        deduplicate_patterns(
+            device_map_patterns(
+                expand_patterns(
+                    patterns,
+                    working_directory=working_directory,
+                    skip_paths=skip_paths,
+                )
+            )
+        )
+    )

borgmatic/actions/recreate.py

@@ -1,12 +1,18 @@
 import logging
+import subprocess
 
+import borgmatic.borg.info
 import borgmatic.borg.recreate
+import borgmatic.borg.repo_list
 import borgmatic.config.validate
-from borgmatic.actions.create import collect_patterns, process_patterns
+from borgmatic.actions.pattern import collect_patterns, process_patterns
 
 logger = logging.getLogger(__name__)
 
 
+BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS = 30
+
+
 def run_recreate(
     repository,
     config,
@@ -32,22 +38,47 @@ def run_recreate(
             collect_patterns(config), borgmatic.config.paths.get_working_directory(config)
         )
 
-        borgmatic.borg.recreate.recreate_archive(
+        archive = borgmatic.borg.repo_list.resolve_archive_name(
             repository['path'],
-            borgmatic.borg.repo_list.resolve_archive_name(
-                repository['path'],
-                recreate_arguments.archive,
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
+            recreate_arguments.archive,
             config,
             local_borg_version,
-            recreate_arguments,
             global_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
-            patterns=processed_patterns,
+            local_path,
+            remote_path,
         )
+
+        if archive and archive.endswith('.recreate'):
+            if recreate_arguments.archive == 'latest':
+                raise ValueError(
+                    f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.'
+                )
+            else:
+                raise ValueError(
+                    f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.'
+                )
+
+        try:
+            borgmatic.borg.recreate.recreate_archive(
+                repository['path'],
+                archive,
+                config,
+                local_borg_version,
+                recreate_arguments,
+                global_arguments,
+                local_path=local_path,
+                remote_path=remote_path,
+                patterns=processed_patterns,
+            )
+        except subprocess.CalledProcessError as error:
+            if error.returncode == BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS:
+                if recreate_arguments.target:
+                    raise ValueError(
+                        f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.'
+                    )
+                elif archive:
+                    raise ValueError(
+                        f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.'
+                    )
+
+            raise

borgmatic/borg/create.py

@@ -1,9 +1,7 @@
-import itertools
 import logging
 import os
 import pathlib
 import stat
-import tempfile
 import textwrap
 
 import borgmatic.borg.pattern
@@ -20,76 +18,6 @@ from borgmatic.execute import (
 logger = logging.getLogger(__name__)
 
 
-def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
-    '''
-    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
-    temporary file in the given borgmatic runtime directory and return the file object so it can
-    continue to exist on disk as long as the caller needs it.
-
-    If an optional open pattern file is given, append to it instead of making a new temporary file.
-    Return None if no patterns are provided.
-    '''
-    if not patterns:
-        return None
-
-    if patterns_file is None:
-        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
-        operation_name = 'Writing'
-    else:
-        patterns_file.write('\n')
-        operation_name = 'Appending'
-
-    patterns_output = '\n'.join(
-        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
-        for pattern in patterns
-    )
-    logger.debug(f'{operation_name} patterns to {patterns_file.name}:\n{patterns_output}')
-
-    patterns_file.write(patterns_output)
-    patterns_file.flush()
-
-    return patterns_file
-
-
-def make_exclude_flags(config):
-    '''
-    Given a configuration dict with various exclude options, return the corresponding Borg flags as
-    a tuple.
-    '''
-    caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
-    if_present_flags = tuple(
-        itertools.chain.from_iterable(
-            ('--exclude-if-present', if_present)
-            for if_present in config.get('exclude_if_present', ())
-        )
-    )
-    keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
-    exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
-
-    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
-
-
-def make_list_filter_flags(local_borg_version, dry_run):
-    '''
-    Given the local Borg version and whether this is a dry run, return the corresponding flags for
-    passing to "--list --filter". The general idea is that excludes are shown for a dry run or when
-    the verbosity is debug.
-    '''
-    base_flags = 'AME'
-    show_excludes = logger.isEnabledFor(logging.DEBUG)
-
-    if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
-        if show_excludes or dry_run:
-            return f'{base_flags}+-'
-        else:
-            return base_flags
-
-    if show_excludes:
-        return f'{base_flags}x-'
-    else:
-        return f'{base_flags}-'
-
-
 def special_file(path, working_directory=None):
     '''
     Return whether the given path is a special file (character device, block device, or named pipe
@@ -141,7 +69,7 @@ def collect_special_file_paths(
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
     # files including any named pipe we've created. And omit "--filter" because that can break the
-    # paths output parsing below such that path lines no longer start with th expected "- ".
+    # paths output parsing below such that path lines no longer start with the expected "- ".
     paths_output = execute_command_and_capture_output(
         flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
         + ('--dry-run', '--list'),
@@ -182,24 +110,6 @@ def collect_special_file_paths(
     )
 
 
-def check_all_root_patterns_exist(patterns):
-    '''
-    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
-    paths exist. If any don't, raise an exception.
-    '''
-    missing_paths = [
-        pattern.path
-        for pattern in patterns
-        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
-        if not os.path.exists(pattern.path)
-    ]
-
-    if missing_paths:
-        raise ValueError(
-            f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}"
-        )
-
-
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
 
 
@@ -224,9 +134,11 @@ def make_base_create_command(
     arguments, open pattern file handle).
     '''
     if config.get('source_directories_must_exist', False):
-        check_all_root_patterns_exist(patterns)
+        borgmatic.borg.pattern.check_all_root_patterns_exist(patterns)
 
-    patterns_file = write_patterns_file(patterns, borgmatic_runtime_directory)
+    patterns_file = borgmatic.borg.pattern.write_patterns_file(
+        patterns, borgmatic_runtime_directory
+    )
     checkpoint_interval = config.get('checkpoint_interval', None)
     checkpoint_volume = config.get('checkpoint_volume', None)
     chunker_params = config.get('chunker_params', None)
@@ -235,7 +147,7 @@ def make_base_create_command(
     upload_buffer_size = config.get('upload_buffer_size', None)
     umask = config.get('umask', None)
     lock_wait = config.get('lock_wait', None)
-    list_filter_flags = make_list_filter_flags(local_borg_version, dry_run)
+    list_filter_flags = flags.make_list_filter_flags(local_borg_version, dry_run)
     files_cache = config.get('files_cache')
     archive_name_format = config.get(
         'archive_name_format', flags.get_default_archive_name_format(local_borg_version)
@@ -270,7 +182,7 @@ def make_base_create_command(
         tuple(local_path.split(' '))
         + ('create',)
         + (('--patterns-from', patterns_file.name) if patterns_file else ())
-        + make_exclude_flags(config)
+        + flags.make_exclude_flags(config)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
@@ -329,7 +241,7 @@ def make_base_create_command(
             logger.warning(
                 f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
             )
-            patterns_file = write_patterns_file(
+            patterns_file = borgmatic.borg.pattern.write_patterns_file(
                 tuple(
                     borgmatic.borg.pattern.Pattern(
                         special_file_path,

borgmatic/borg/flags.py

@@ -197,3 +197,42 @@ def omit_flag_and_value(arguments, flag):
         if flag not in (previous_argument, argument)
         if not argument.startswith(f'{flag}=')
     )
+
+
+def make_exclude_flags(config):
+    '''
+    Given a configuration dict with various exclude options, return the corresponding Borg flags as
+    a tuple.
+    '''
+    caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
+    if_present_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-if-present', if_present)
+            for if_present in config.get('exclude_if_present', ())
+        )
+    )
+    keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
+    exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
+
+    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
+
+
+def make_list_filter_flags(local_borg_version, dry_run):
+    '''
+    Given the local Borg version and whether this is a dry run, return the corresponding flags for
+    passing to "--list --filter". The general idea is that excludes are shown for a dry run or when
+    the verbosity is debug.
+    '''
+    base_flags = 'AME'
+    show_excludes = logger.isEnabledFor(logging.DEBUG)
+
+    if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
+        if show_excludes or dry_run:
+            return f'{base_flags}+-'
+        else:
+            return base_flags
+
+    if show_excludes:
+        return f'{base_flags}x-'
+    else:
+        return f'{base_flags}-'

borgmatic/borg/pattern.py

@@ -1,5 +1,12 @@
 import collections
 import enum
+import logging
+import os
+import tempfile
+
+import borgmatic.borg.pattern
+
+logger = logging.getLogger(__name__)
 
 
 # See https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns
@@ -48,3 +55,52 @@ Pattern = collections.namedtuple(
         Pattern_source.HOOK,
     ),
 )
+
+
+def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
+    '''
+    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
+    temporary file in the given borgmatic runtime directory and return the file object so it can
+    continue to exist on disk as long as the caller needs it.
+
+    If an optional open pattern file is given, append to it instead of making a new temporary file.
+    Return None if no patterns are provided.
+    '''
+    if not patterns:
+        return None
+
+    if patterns_file is None:
+        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
+        operation_name = 'Writing'
+    else:
+        patterns_file.write('\n')
+        operation_name = 'Appending'
+
+    patterns_output = '\n'.join(
+        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
+        for pattern in patterns
+    )
+    logger.debug(f'{operation_name} patterns to {patterns_file.name}:\n{patterns_output}')
+
+    patterns_file.write(patterns_output)
+    patterns_file.flush()
+
+    return patterns_file
+
+
+def check_all_root_patterns_exist(patterns):
+    '''
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
+    paths exist. If any don't, raise an exception.
+    '''
+    missing_paths = [
+        pattern.path
+        for pattern in patterns
+        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+        if not os.path.exists(pattern.path)
+    ]
+
+    if missing_paths:
+        raise ValueError(
+            f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}"
+        )

borgmatic/borg/recreate.py

@@ -6,7 +6,7 @@ import borgmatic.borg.feature
 import borgmatic.config.paths
 import borgmatic.execute
 from borgmatic.borg import flags
-from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, write_patterns_file
+from borgmatic.borg.pattern import write_patterns_file
 
 logger = logging.getLogger(__name__)
 
@@ -29,9 +29,10 @@ def recreate_archive(
     arguments.
     '''
     lock_wait = config.get('lock_wait', None)
-    exclude_flags = make_exclude_flags(config)
+    exclude_flags = flags.make_exclude_flags(config)
     compression = config.get('compression', None)
     chunker_params = config.get('chunker_params', None)
+
     # Available recompress MODES: "if-different", "always", "never" (default)
     recompress = config.get('recompress', None)
 
@@ -52,7 +53,7 @@ def recreate_archive(
             (
                 '--list',
                 '--filter',
-                make_list_filter_flags(local_borg_version, global_arguments.dry_run),
+                flags.make_list_filter_flags(local_borg_version, global_arguments.dry_run),
             )
             if config.get('list_details')
             else ()

borgmatic/commands/arguments.py

@@ -357,7 +357,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name):
     if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments:
         return
 
-    pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$')
+    pattern = re.compile('^--' + flag_name.replace('[0]', r'\[\d+\]').replace('.', r'\.') + '$')
 
     try:
         # Find an existing list index flag (and its action) corresponding to the given flag name.
@@ -587,7 +587,7 @@ def make_parsers(schema, unparsed_arguments):
         '--no-environment-interpolation',
         dest='resolve_env',
         action='store_false',
-        help='Do not resolve environment variables in configuration file',
+        help='Do not resolve environment variables in configuration files',
     )
     global_group.add_argument(
         '--bash-completion',

borgmatic/commands/borgmatic.py

@@ -279,8 +279,19 @@ def run_configuration(config_filename, config, config_paths, arguments):
                             encountered_error = error
                             error_repository = repository
 
+                # Re-raise any error, so that the Monitoring_hooks context manager wrapping this
+                # code can see the error and act accordingly. Do this here rather than as soon as
+                # the error is encountered so that an error with one repository doesn't prevent
+                # other repositories from running.
+                if encountered_error:
+                    raise encountered_error
+
     except (OSError, CalledProcessError, ValueError) as error:
-        yield from log_error_records('Error running configuration', error)
+        # No need to repeat logging of the error if it was already logged above.
+        if error_repository:
+            yield from log_error_records('Error running configuration')
+        else:
+            yield from log_error_records('Error running configuration', error)
 
         encountered_error = error
 
@@ -290,7 +301,10 @@ def run_configuration(config_filename, config, config_paths, arguments):
     try:
         command.execute_hooks(
             command.filter_hooks(
-                config.get('commands'), after='error', action_names=arguments.keys()
+                config.get('commands'),
+                after='error',
+                action_names=arguments.keys(),
+                state_names=['fail'],
             ),
             config.get('umask'),
             borgmatic.config.paths.get_working_directory(config),
@@ -356,7 +370,7 @@ def run_actions(
         **hook_context,
     ):
         for action_name, action_arguments in arguments.items():
-            if action_name == 'global':
+            if action_name == 'global' or action_name in skip_actions:
                 continue
 
             with borgmatic.hooks.command.Before_after_hooks(
@@ -365,10 +379,10 @@ def run_actions(
                 umask=config.get('umask'),
                 working_directory=borgmatic.config.paths.get_working_directory(config),
                 dry_run=global_arguments.dry_run,
-                action_names=arguments.keys(),
+                action_names=(action_name,),
                 **hook_context,
             ):
-                if action_name == 'repo-create' and action_name not in skip_actions:
+                if action_name == 'repo-create':
                     borgmatic.actions.repo_create.run_repo_create(
                         repository,
                         config,
@@ -378,7 +392,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'transfer' and action_name not in skip_actions:
+                elif action_name == 'transfer':
                     borgmatic.actions.transfer.run_transfer(
                         repository,
                         config,
@@ -388,7 +402,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'create' and action_name not in skip_actions:
+                elif action_name == 'create':
                     yield from borgmatic.actions.create.run_create(
                         config_filename,
                         repository,
@@ -401,7 +415,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'recreate' and action_name not in skip_actions:
+                elif action_name == 'recreate':
                     borgmatic.actions.recreate.run_recreate(
                         repository,
                         config,
@@ -411,7 +425,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'prune' and action_name not in skip_actions:
+                elif action_name == 'prune':
                     borgmatic.actions.prune.run_prune(
                         config_filename,
                         repository,
@@ -423,7 +437,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'compact' and action_name not in skip_actions:
+                elif action_name == 'compact':
                     borgmatic.actions.compact.run_compact(
                         config_filename,
                         repository,
@@ -435,7 +449,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'check' and action_name not in skip_actions:
+                elif action_name == 'check':
                     if checks.repository_enabled_for_checks(repository, config):
                         borgmatic.actions.check.run_check(
                             config_filename,
@@ -447,7 +461,7 @@ def run_actions(
                             local_path,
                             remote_path,
                         )
-                elif action_name == 'extract' and action_name not in skip_actions:
+                elif action_name == 'extract':
                     borgmatic.actions.extract.run_extract(
                         config_filename,
                         repository,
@@ -458,7 +472,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'export-tar' and action_name not in skip_actions:
+                elif action_name == 'export-tar':
                     borgmatic.actions.export_tar.run_export_tar(
                         repository,
                         config,
@@ -468,7 +482,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'mount' and action_name not in skip_actions:
+                elif action_name == 'mount':
                     borgmatic.actions.mount.run_mount(
                         repository,
                         config,
@@ -478,7 +492,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'restore' and action_name not in skip_actions:
+                elif action_name == 'restore':
                     borgmatic.actions.restore.run_restore(
                         repository,
                         config,
@@ -488,7 +502,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'repo-list' and action_name not in skip_actions:
+                elif action_name == 'repo-list':
                     yield from borgmatic.actions.repo_list.run_repo_list(
                         repository,
                         config,
@@ -498,7 +512,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'list' and action_name not in skip_actions:
+                elif action_name == 'list':
                     yield from borgmatic.actions.list.run_list(
                         repository,
                         config,
@@ -508,7 +522,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'repo-info' and action_name not in skip_actions:
+                elif action_name == 'repo-info':
                     yield from borgmatic.actions.repo_info.run_repo_info(
                         repository,
                         config,
@@ -518,7 +532,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'info' and action_name not in skip_actions:
+                elif action_name == 'info':
                     yield from borgmatic.actions.info.run_info(
                         repository,
                         config,
@@ -528,7 +542,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'break-lock' and action_name not in skip_actions:
+                elif action_name == 'break-lock':
                     borgmatic.actions.break_lock.run_break_lock(
                         repository,
                         config,
@@ -538,7 +552,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'export' and action_name not in skip_actions:
+                elif action_name == 'export':
                     borgmatic.actions.export_key.run_export_key(
                         repository,
                         config,
@@ -548,7 +562,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'import' and action_name not in skip_actions:
+                elif action_name == 'import':
                     borgmatic.actions.import_key.run_import_key(
                         repository,
                         config,
@@ -558,7 +572,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'change-passphrase' and action_name not in skip_actions:
+                elif action_name == 'change-passphrase':
                     borgmatic.actions.change_passphrase.run_change_passphrase(
                         repository,
                         config,
@@ -568,7 +582,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'delete' and action_name not in skip_actions:
+                elif action_name == 'delete':
                     borgmatic.actions.delete.run_delete(
                         repository,
                         config,
@@ -578,7 +592,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'repo-delete' and action_name not in skip_actions:
+                elif action_name == 'repo-delete':
                     borgmatic.actions.repo_delete.run_repo_delete(
                         repository,
                         config,
@@ -588,7 +602,7 @@ def run_actions(
                         local_path,
                         remote_path,
                     )
-                elif action_name == 'borg' and action_name not in skip_actions:
+                elif action_name == 'borg':
                     borgmatic.actions.borg.run_borg(
                         repository,
                         config,
@@ -879,23 +893,36 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
         return
 
     try:
+        seen_command_hooks = []
+
         for config_filename, config in configs.items():
-            command.execute_hooks(
-                command.filter_hooks(
-                    config.get('commands'), before='everything', action_names=arguments.keys()
+            command_hooks = command.filter_hooks(
+                tuple(
+                    command_hook
+                    for command_hook in config.get('commands', ())
+                    if command_hook not in seen_command_hooks
                 ),
-                config.get('umask'),
-                borgmatic.config.paths.get_working_directory(config),
-                arguments['global'].dry_run,
-                configuration_filename=config_filename,
-                log_file=log_file_path or '',
+                before='everything',
+                action_names=arguments.keys(),
             )
+
+            if command_hooks:
+                command.execute_hooks(
+                    command_hooks,
+                    config.get('umask'),
+                    borgmatic.config.paths.get_working_directory(config),
+                    arguments['global'].dry_run,
+                    configuration_filename=config_filename,
+                    log_file=log_file_path or '',
+                )
+                seen_command_hooks += list(command_hooks)
     except (CalledProcessError, ValueError, OSError) as error:
         yield from log_error_records('Error running before everything hook', error)
         return
 
     # Execute the actions corresponding to each configuration file.
     json_results = []
+    encountered_error = False
 
     for config_filename, config in configs.items():
         with Log_prefix(config_filename):
@@ -906,6 +933,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
             )
 
             if error_logs:
+                encountered_error = True
                 yield from log_error_records('An error occurred')
                 yield from error_logs
             else:
@@ -928,23 +956,37 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
                 local_path=get_local_path(configs),
             )
         except (CalledProcessError, OSError) as error:
+            encountered_error = True
             yield from log_error_records('Error unmounting mount point', error)
 
     if json_results:
         sys.stdout.write(json.dumps(json_results))
 
     try:
+        seen_command_hooks = []
+
         for config_filename, config in configs.items():
-            command.execute_hooks(
-                command.filter_hooks(
-                    config.get('commands'), after='everything', action_names=arguments.keys()
+            command_hooks = command.filter_hooks(
+                tuple(
+                    command_hook
+                    for command_hook in config.get('commands', ())
+                    if command_hook not in seen_command_hooks
                 ),
-                config.get('umask'),
-                borgmatic.config.paths.get_working_directory(config),
-                arguments['global'].dry_run,
-                configuration_filename=config_filename,
-                log_file=log_file_path or '',
+                after='everything',
+                action_names=arguments.keys(),
+                state_names=['fail' if encountered_error else 'finish'],
             )
+
+            if command_hooks:
+                command.execute_hooks(
+                    command_hooks,
+                    config.get('umask'),
+                    borgmatic.config.paths.get_working_directory(config),
+                    arguments['global'].dry_run,
+                    configuration_filename=config_filename,
+                    log_file=log_file_path or '',
+                )
+                seen_command_hooks += list(command_hooks)
     except (CalledProcessError, ValueError, OSError) as error:
         yield from log_error_records('Error running after everything hook', error)
 
@@ -1040,7 +1082,7 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         config_filenames,
         arguments,
         global_arguments.overrides,
-        resolve_env=global_arguments.resolve_env and not validate,
+        resolve_env=global_arguments.resolve_env and not arguments.get('validate'),
     )
 
     # Use the helper function to check and show help on no arguments, passing the preloaded configs

borgmatic/config/schema.yaml

@@ -1225,6 +1225,26 @@ properties:
                               particular actions listed here. Defaults to
                               running for all actions.
                           example: [create, prune, compact, check]
+                      states:
+                          type: array
+                          items:
+                              type: string
+                              enum:
+                                  - finish
+                                  - fail
+                          description: |
+                              Only trigger the hook if borgmatic encounters one
+                              of the states (execution results) listed here,
+                              where:
+                               * "finish": No errors occurred.
+                               * "fail": An error occurred.
+                              This state is evaluated only for the scope of the
+                              configured "action", "repository", etc., rather
+                              than for the entire borgmatic run. Only available
+                              for "after" hooks. Defaults to running the hook
+                              for all states.
+                          example:
+                              - finish
                       run:
                           type: array
                           items:

borgmatic/hooks/command.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import os
 import re
@@ -12,6 +13,19 @@ logger = logging.getLogger(__name__)
 
 
 SOFT_FAIL_EXIT_CODE = 75
+BORG_PLACEHOLDERS = {
+    '{hostname}',
+    '{fqdn}',
+    '{reverse-fqdn}',
+    '{now}',
+    '{utcnow}',
+    '{user}',
+    '{pid}',
+    '{borgversion}',
+    '{borgmajor}',
+    '{borgminor}',
+    '{borgpatch}',
+}
 
 
 def interpolate_context(hook_description, command, context):
@@ -22,10 +36,13 @@ def interpolate_context(hook_description, command, context):
     for name, value in context.items():
         command = command.replace(f'{{{name}}}', shlex.quote(str(value)))
 
-    for unsupported_variable in re.findall(r'{\w+}', command):
-        logger.warning(
-            f"Variable '{unsupported_variable}' is not supported in {hook_description} hook"
-        )
+    for unsupported_variable in re.findall(r'\{\w+\}', command):
+        # Warn about variables unknown to borgmatic, but don't warn if the variable name happens to
+        # be a Borg placeholder, as Borg should hopefully consume it.
+        if unsupported_variable not in BORG_PLACEHOLDERS:
+            logger.warning(
+                f'Variable "{unsupported_variable}" is not supported in the {hook_description} hook'
+            )
 
     return command
 
@@ -46,21 +63,25 @@ def make_environment(current_environment, sys_module=sys):
     return environment
 
 
-def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_names=None):
+def filter_hooks(command_hooks, before=None, after=None, action_names=None, state_names=None):
     '''
     Given a sequence of command hook dicts from configuration and one or more filters (before name,
-    after name, calling hook name, or a sequence of action names), filter down the command hooks to
-    just the ones that match the given filters.
+    after name, a sequence of action names, and/or a sequence of execution result state names),
+    filter down the command hooks to just the ones that match the given filters.
     '''
     return tuple(
         hook_config
         for hook_config in command_hooks or ()
         for config_action_names in (hook_config.get('when'),)
+        for config_state_names in (hook_config.get('states'),)
         if before is None or hook_config.get('before') == before
         if after is None or hook_config.get('after') == after
         if action_names is None
         or config_action_names is None
         or set(config_action_names or ()).intersection(set(action_names))
+        if state_names is None
+        or config_state_names is None
+        or set(config_state_names or ()).intersection(set(state_names))
     )
 
 
@@ -81,11 +102,14 @@ def execute_hooks(command_hooks, umask, working_directory, dry_run, **context):
 
     for hook_config in command_hooks:
         commands = hook_config.get('run')
+        when_description = (
+            f"{'/'.join(hook_config.get('when'))} " if hook_config.get('when') else ''
+        )
 
         if 'before' in hook_config:
-            description = f'before {hook_config.get("before")}'
+            description = f'before {when_description}{hook_config.get("before")}'
         elif 'after' in hook_config:
-            description = f'after {hook_config.get("after")}'
+            description = f'after {when_description}{hook_config.get("after")}'
         else:
             raise ValueError(f'Invalid hook configuration: {hook_config}')
 
@@ -139,7 +163,7 @@ class Before_after_hooks:
            before_after='do_stuff',
            umask=config.get('umask'),
            dry_run=dry_run,
-           hook_name='myhook',
+           action_names=['create'],
        ):
             do()
             some()
@@ -156,22 +180,20 @@ class Before_after_hooks:
         umask,
         working_directory,
         dry_run,
-        hook_name=None,
         action_names=None,
         **context,
     ):
         '''
         Given a sequence of command hook configuration dicts, the before/after name, a umask to run
-        commands with, a working directory to run commands with, a dry run flag, the name of the
-        calling hook, a sequence of action names, and any context for the executed commands, save
-        those data points for use below.
+        commands with, a working directory to run commands with, a dry run flag, a sequence of
+        action names, and any context for the executed commands, save those data points for use
+        below.
         '''
         self.command_hooks = command_hooks
         self.before_after = before_after
         self.umask = umask
         self.working_directory = working_directory
         self.dry_run = dry_run
-        self.hook_name = hook_name
         self.action_names = action_names
         self.context = context
 
@@ -184,7 +206,6 @@ class Before_after_hooks:
                 borgmatic.hooks.command.filter_hooks(
                     self.command_hooks,
                     before=self.before_after,
-                    hook_name=self.hook_name,
                     action_names=self.action_names,
                 ),
                 self.umask,
@@ -194,11 +215,11 @@ class Before_after_hooks:
             )
         except (OSError, subprocess.CalledProcessError) as error:
             if considered_soft_failure(error):
-                return
+                raise
 
             # Trigger the after hook manually, since raising here will prevent it from being run
             # otherwise.
-            self.__exit__(None, None, None)
+            self.__exit__(exception_type=type(error), exception=error, traceback=None)
 
             raise ValueError(f'Error running before {self.before_after} hook: {error}')
 
@@ -211,8 +232,8 @@ class Before_after_hooks:
                 borgmatic.hooks.command.filter_hooks(
                     self.command_hooks,
                     after=self.before_after,
-                    hook_name=self.hook_name,
                     action_names=self.action_names,
+                    state_names=['fail' if exception_type else 'finish'],
                 ),
                 self.umask,
                 self.working_directory,
@@ -221,16 +242,20 @@ class Before_after_hooks:
             )
         except (OSError, subprocess.CalledProcessError) as error:
             if considered_soft_failure(error):
-                return
+                raise
 
             raise ValueError(f'Error running after {self.before_after} hook: {error}')
 
 
+@functools.cache
 def considered_soft_failure(error):
     '''
     Given a configuration filename and an exception object, return whether the exception object
     represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so,
     that indicates that the error is a "soft failure", and should not result in an error.
+
+    The results of this function are cached so that it can be called multiple times without logging
+    multiple times.
     '''
     exit_code = getattr(error, 'returncode', None)
     if exit_code is None:

borgmatic/hooks/data_source/lvm.py

@@ -407,7 +407,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
                 continue
 
         if not dry_run:
-            shutil.rmtree(snapshots_directory)
+            shutil.rmtree(snapshots_directory, ignore_errors=True)
 
     # Delete snapshots.
     lvremove_command = hook_config.get('lvremove_command', 'lvremove')

borgmatic/hooks/data_source/snapshot.py

@@ -8,9 +8,9 @@ def get_contained_patterns(parent_directory, candidate_patterns):
     '''
     Given a parent directory and a set of candidate patterns potentially inside it, get the subset
     of contained patterns for which the parent directory is actually the parent, a grandparent, the
-    very same directory, etc. The idea is if, say, /var/log and /var/lib are candidate pattern
-    paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then
-    /var is what we want to snapshot.
+    very same directory, etc. The idea is if, say, "/var/log" and "/var/lib" are candidate pattern
+    paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at "/var", then
+    "/var" is what we want to snapshot.
 
     If a parent directory and a candidate pattern are on different devices, skip the pattern. That's
     because any snapshot of a parent directory won't actually include "contained" directories if
@@ -18,8 +18,8 @@ def get_contained_patterns(parent_directory, candidate_patterns):
 
     For this function to work, a candidate pattern path can't have any globs or other non-literal
     characters in the initial portion of the path that matches the parent directory. For instance, a
-    parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a
-    pattern path like /v*/log/*/data.
+    parent directory of "/var" would match a candidate pattern path of "/var/log/*/data", but not a
+    pattern path like "/v*/log/*/data".
 
     The one exception is that if a regular expression pattern path starts with "^", that will get
     stripped off for purposes of matching against a parent directory.

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -55,6 +55,25 @@ commands:
           - "echo Backup: start"
 ```
 
+By default, an `after` command hook runs even if an error occurs in the
+corresponding `before` hook or between those two hooks. This allows you to
+perform cleanup steps that correspond to `before` preparation commands—even when
+something goes wrong. You may notice that this is a departure from the way that
+the deprecated `after_*` hooks worked in borgmatic prior to version 2.0.0.
+
+<span class="minilink minilink-addedin">New in version 2.0.3</span> You can
+customize this behavior with the `states` option. For instance, here's an
+example of an `after` hook that only triggers on success and not on error:
+
+```yaml
+commands:
+    - after: action
+      when: [create]
+      states: [finish]
+      run:
+          - echo "After successful create!"
+```
+
 Each command in the `commands:` list has the following options:
 
  * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of:
@@ -64,17 +83,17 @@ Each command in the `commands:` list has the following options:
     * `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`.
     * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook.
  * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions.
+ * `states`: <span class="minilink minilink-addedin">New in version 2.0.3</span> Only trigger the hook if borgmatic encounters one of the states (execution results) listed here. This state is evaluated only for the scope of the configured `action`, `repository`, etc., rather than for the entire borgmatic run. Only available for `after` hooks. Defaults to running the hook for all states. One or more of:
+    * `finish`: No errors occurred.
+    * `fail`: An error occurred.
  * `run`: List of one or more shell commands or scripts to run when this command hook is triggered.
 
-An `after` command hook runs even if an error occurs in the corresponding
-`before` hook or between those two hooks. This allows you to perform cleanup
-steps that correspond to `before` preparation commands—even when something goes
-wrong. This is a departure from the way that the deprecated `after_*` hooks
-worked in borgmatic prior to version 2.0.0.
+When command hooks run, they respect the `working_directory` option if it is
+configured, meaning that the hook commands are run in that directory.
 
-Additionally, when command hooks run, they respect the `working_directory`
-option if it is configured, meaning that the hook commands are run in that
-directory.
+<span class="minilink minilink-addedin">New in version 2.0.4</span>If the exact
+same `everything` command hook is present in multiple configuration files,
+borgmatic only runs it once.
 
 
 ### Order of execution
@@ -97,11 +116,20 @@ borgmatic for the `create` and `prune` actions. Here's the order of execution:
             * Run `after: action` hooks for `prune`.
         * Run `after: repository` hooks (for the first repository).
     * Run `after: configuration` hooks (from the first configuration file).
+    * Run `after: error` hooks (if an error occurs).
  * Run `after: everything` hooks (from all configuration files).
 
 This same order of execution extends to multiple repositories and/or
 configuration files.
 
+Based on the above, you can see the difference between, say, an `after: action`
+hook with `states: [fail]` and an `after: error` hook. The `after: action hook`
+runs immediately after the create action fails for a particular repository—so
+before any subsequent actions for that repository or other repositories even
+have a chance to run. Whereas the `after: error` hook doesn't run until all
+actions for—and repositories in—a configuration file have had a chance to
+execute.
+
 
 ### Deprecated command hooks
 

docs/how-to/snapshot-your-filesystems.md

@@ -116,6 +116,27 @@ you can use the standard
 extract them.
 
 
+#### ZFS performance
+
+<span class="minilink minilink-addedin">With Borg version 1.x</span> Because of
+the way that ZFS snapshot paths can change from one borgmatic invocation to the
+next, the [Borg file
+cache](https://borgbackup.readthedocs.io/en/stable/internals/data-structures.html#cache)
+may not get cache hits on snapshotted files. This makes backing up ZFS snapshots
+a little slower than non-snapshotted files that have consistent paths. You can
+mitigate this by setting a fixed [runtime
+directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory)
+(that's not located in `/tmp`). This allows borgmatic to use a consistent
+snapshot path from one run to the next, thereby resulting in Borg files cache
+hits.
+
+<span class="minilink minilink-addedin">With Borg version 2.x</span> Snapshotted
+files should get cache hits regardless of whether their paths change, because
+Borg 2.x is smarter about how it looks up file paths in its cache—it constructs
+the cache key with the path *as it's seen in the archive* (which is consistent
+across runs) rather than the full absolute source path (which can change).
+
+
 ### Btrfs
 
 <span class="minilink minilink-addedin">New in version 1.9.4</span> <span
@@ -197,6 +218,22 @@ action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) to extract
 them.
 
 
+#### Btrfs performance
+
+<span class="minilink minilink-addedin">With Borg version 1.x</span> Because of
+the way that Btrfs snapshot paths change from one borgmatic invocation to the
+next, the [Borg file
+cache](https://borgbackup.readthedocs.io/en/stable/internals/data-structures.html#cache)
+will never get cache hits on snapshotted files. This makes backing up Btrfs
+snapshots a little slower than non-snapshotted files that have consistent paths.
+
+<span class="minilink minilink-addedin">With Borg version 2.x</span> Even
+snapshotted files should get cache hits, because Borg 2.x is smarter about how
+it looks up file paths in its cache—it constructs the cache key with the path
+*as it's seen in the archive* (which is consistent across runs) rather than the
+full absolute source path (which changes).
+
+
 ### LVM
 
 <span class="minilink minilink-addedin">New in version 1.9.4</span> <span
@@ -311,3 +348,24 @@ Logical volume snapshots are stored in a Borg archive as normal files, so
 you can use the standard
 [extract action](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) to
 extract them.
+
+
+#### LVM performance
+
+<span class="minilink minilink-addedin">With Borg version 1.x</span> Because of
+the way that LVM snapshot paths can change from one borgmatic invocation to the
+next, the [Borg file
+cache](https://borgbackup.readthedocs.io/en/stable/internals/data-structures.html#cache)
+may not get cache hits on snapshotted files. This makes backing up LVM snapshots
+a little slower than non-snapshotted files that have consistent paths. You can
+mitigate this by setting a fixed [runtime
+directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory)
+(that's not located in `/tmp`). This allows borgmatic to use a consistent
+snapshot path from one run to the next, thereby resulting in Borg files cache
+hits.
+
+<span class="minilink minilink-addedin">With Borg version 2.x</span> Snapshotted
+files should get cache hits regardless of whether their paths change, because
+Borg 2.x is smarter about how it looks up file paths in its cache—it constructs
+the cache key with the path *as it's seen in the archive* (which is consistent
+across runs) rather than the full absolute source path (which can change).

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "2.0.0"
+version = "2.0.4"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]

tests/end-to-end/commands/fake_btrfs.py

@@ -53,7 +53,7 @@ def save_snapshots(snapshot_paths):
 
 
 def print_subvolume_list(arguments, snapshot_paths):
-    assert arguments.subvolume_path == '/mnt/subvolume'
+    assert arguments.subvolume_path == '/e2e/mnt/subvolume'
 
     if not arguments.snapshots_only:
         for line in BUILTIN_SUBVOLUME_LIST_LINES:

tests/end-to-end/commands/fake_findmnt.py

@@ -14,7 +14,7 @@ def parse_arguments(*unparsed_arguments):
 BUILTIN_FILESYSTEM_MOUNT_OUTPUT = '''{
        "filesystems": [
           {
-             "target": "/mnt/subvolume",
+             "target": "/e2e/mnt/subvolume",
              "source": "/dev/loop0",
              "fstype": "btrfs",
              "options": "rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/"

tests/end-to-end/commands/fake_lsblk.py

@@ -20,7 +20,7 @@ BUILTIN_BLOCK_DEVICES = {
         {
             'name': 'vgroup-lvolume',
             'path': '/dev/mapper/vgroup-lvolume',
-            'mountpoint': '/mnt/lvolume',
+            'mountpoint': '/e2e/mnt/lvolume',
             'type': 'lvm',
         },
         {

tests/end-to-end/commands/fake_zfs.py

@@ -23,20 +23,20 @@ def parse_arguments(*unparsed_arguments):
 
 BUILTIN_DATASETS = (
     {
-        'name': 'pool',
+        'name': 'e2e/pool',
         'used': '256K',
         'avail': '23.7M',
         'refer': '25K',
         'canmount': 'on',
-        'mountpoint': '/pool',
+        'mountpoint': '/e2e/pool',
     },
     {
-        'name': 'pool/dataset',
+        'name': 'e2e/pool/dataset',
         'used': '256K',
         'avail': '23.7M',
         'refer': '25K',
         'canmount': 'on',
-        'mountpoint': '/pool/dataset',
+        'mountpoint': '/e2e/pool/dataset',
     },
 )
 

tests/end-to-end/hooks/data_source/test_btrfs.py

@@ -16,10 +16,10 @@ def generate_configuration(config_path, repository_path):
         open(config_path)
         .read()
         .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
-        .replace('- path: /mnt/backup', '')
+        .replace('- path: /e2e/mnt/backup', '')
         .replace('label: local', '')
         .replace('- /home', f'- {config_path}')
-        .replace('- /etc', '- /mnt/subvolume/subdir')
+        .replace('- /etc', '- /e2e/mnt/subvolume/subdir')
         .replace('- /var/log/syslog*', '')
         + 'encryption_passphrase: "test"\n'
         + 'btrfs:\n'
@@ -51,11 +51,11 @@ def test_btrfs_create_and_list():
             f'borgmatic --config {config_path} list --archive latest'.split(' ')
         ).decode(sys.stdout.encoding)
 
-        assert 'mnt/subvolume/subdir/file.txt' in output
+        assert 'e2e/mnt/subvolume/subdir/file.txt' in output
 
         # Assert that the snapshot has been deleted.
         assert not subprocess.check_output(
-            'python3 /app/tests/end-to-end/commands/fake_btrfs.py subvolume list -s /mnt/subvolume'.split(
+            'python3 /app/tests/end-to-end/commands/fake_btrfs.py subvolume list -s /e2e/mnt/subvolume'.split(
                 ' '
             )
         )

tests/end-to-end/hooks/data_source/test_lvm.py

@@ -17,10 +17,10 @@ def generate_configuration(config_path, repository_path):
         open(config_path)
         .read()
         .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
-        .replace('- path: /mnt/backup', '')
+        .replace('- path: /e2e/mnt/backup', '')
         .replace('label: local', '')
         .replace('- /home', f'- {config_path}')
-        .replace('- /etc', '- /mnt/lvolume/subdir')
+        .replace('- /etc', '- /e2e/mnt/lvolume/subdir')
         .replace('- /var/log/syslog*', '')
         + 'encryption_passphrase: "test"\n'
         + 'lvm:\n'
@@ -56,7 +56,7 @@ def test_lvm_create_and_list():
             f'borgmatic --config {config_path} list --archive latest'.split(' ')
         ).decode(sys.stdout.encoding)
 
-        assert 'mnt/lvolume/subdir/file.txt' in output
+        assert 'e2e/mnt/lvolume/subdir/file.txt' in output
 
         # Assert that the snapshot has been deleted.
         assert not json.loads(

tests/end-to-end/hooks/data_source/test_zfs.py

@@ -19,7 +19,7 @@ def generate_configuration(config_path, repository_path):
         .replace('- path: /mnt/backup', '')
         .replace('label: local', '')
         .replace('- /home', f'- {config_path}')
-        .replace('- /etc', '- /pool/dataset/subdir')
+        .replace('- /etc', '- /e2e/pool/dataset/subdir')
         .replace('- /var/log/syslog*', '')
         + 'encryption_passphrase: "test"\n'
         + 'zfs:\n'
@@ -45,14 +45,14 @@ def test_zfs_create_and_list():
         )
 
         # Run a create action to exercise ZFS snapshotting and backup.
-        subprocess.check_call(f'borgmatic --config {config_path} create'.split(' '))
+        subprocess.check_call(f'borgmatic -v 2 --config {config_path} create'.split(' '))
 
         # List the resulting archive and assert that the snapshotted files are there.
         output = subprocess.check_output(
             f'borgmatic --config {config_path} list --archive latest'.split(' ')
         ).decode(sys.stdout.encoding)
 
-        assert 'pool/dataset/subdir/file.txt' in output
+        assert 'e2e/pool/dataset/subdir/file.txt' in output
 
         # Assert that the snapshot has been deleted.
         assert not subprocess.check_output(

tests/end-to-end/test_environment_variable.py

@@ -0,0 +1,71 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase environment variable (which will get passed in
+    when the Borg command is called below).
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + '\nencryption_passphrase: "${PASSPHRASE}"'
+        # Disable automatic storage of config files so we can test storage and extraction manually.
+        + '\nbootstrap:\n  store_config_files: false'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_borgmatic_command():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    extract_path = os.path.join(temporary_directory, 'extract')
+
+    original_working_directory = os.getcwd()
+    os.mkdir(extract_path)
+    os.chdir(extract_path)
+    environment = dict(os.environ, **{'PASSPHRASE': 'test'})
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path)
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '),
+            env=environment,
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call(
+            f'borgmatic --config {config_path}'.split(' '),
+            env=environment,
+        )
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --json'.split(' '),
+            env=environment,
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/integration/commands/test_borgmatic.py

@@ -12,3 +12,59 @@ def test_borgmatic_version_matches_news_version():
     news_version = open('NEWS').readline()
 
     assert borgmatic_version == news_version
+
+
+def test_run_configuration_without_error_pings_monitoring_hooks_start_and_finish():
+    config = {'repositories': [{'path': 'foo'}]}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
+    flexmock(module).should_receive('run_actions').and_return([])
+    flexmock(module.dispatch).should_receive('call_hooks')
+    flexmock(module.dispatch).should_receive('call_hooks').with_args(
+        'ping_monitor',
+        config,
+        module.dispatch.Hook_type.MONITORING,
+        'test.yaml',
+        module.monitor.State.START,
+        object,
+        object,
+    ).once()
+    flexmock(module.dispatch).should_receive('call_hooks').with_args(
+        'ping_monitor',
+        config,
+        module.dispatch.Hook_type.MONITORING,
+        'test.yaml',
+        module.monitor.State.FINISH,
+        object,
+        object,
+    ).once()
+
+    list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments))
+
+
+def test_run_configuration_with_action_error_pings_monioring_hooks_start_and_fail():
+    config = {'repositories': [{'path': 'foo'}]}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
+    flexmock(module).should_receive('run_actions').and_raise(OSError)
+    flexmock(module.dispatch).should_receive('call_hooks')
+    flexmock(module.dispatch).should_receive('call_hooks').with_args(
+        'ping_monitor',
+        config,
+        module.dispatch.Hook_type.MONITORING,
+        'test.yaml',
+        module.monitor.State.START,
+        object,
+        object,
+    ).once()
+    flexmock(module.dispatch).should_receive('call_hooks').with_args(
+        'ping_monitor',
+        config,
+        module.dispatch.Hook_type.MONITORING,
+        'test.yaml',
+        module.monitor.State.FAIL,
+        object,
+        object,
+    ).once()
+
+    list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments))

tests/integration/config/test_schema.py

@@ -20,21 +20,28 @@ ACTIONS_MODULE_NAMES_TO_OMIT = {
     'export_key',
     'import_key',
     'json',
+    'pattern',
 }
 ACTIONS_MODULE_NAMES_TO_ADD = {'key', 'umount'}
 
 
-def test_schema_skip_actions_correspond_to_supported_actions():
+def test_schema_actions_correspond_to_supported_actions():
     '''
-    Ensure that the allowed actions in the schema's "skip_actions" option don't drift from
-    borgmatic's actual supported actions.
+    Ensure that the allowed actions in the schema's various options don't drift from borgmatic's
+    actual supported actions.
     '''
     schema = borgmatic.config.load.load_configuration(borgmatic.config.validate.schema_filename())
-    schema_skip_actions = set(schema['properties']['skip_actions']['items']['enum'])
     supported_actions = {
         module.name.replace('_', '-')
         for module in pkgutil.iter_modules(borgmatic.actions.__path__)
         if module.name not in ACTIONS_MODULE_NAMES_TO_OMIT
     }.union(ACTIONS_MODULE_NAMES_TO_ADD)
+    properties = schema['properties']
+    commands_one_of = properties['commands']['items']['oneOf']
 
-    assert schema_skip_actions == supported_actions
+    for schema_actions in (
+        set(properties['skip_actions']['items']['enum']),
+        set(commands_one_of[0]['properties']['when']['items']['enum']),
+        set(commands_one_of[1]['properties']['when']['items']['enum']),
+    ):
+        assert schema_actions == supported_actions

tests/unit/actions/test_check.py

@@ -561,10 +561,10 @@ def test_collect_spot_check_source_paths_parses_borg_output():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')]
     )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
@@ -608,10 +608,10 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')]
     )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
@@ -655,10 +655,10 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')]
     )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
@@ -702,10 +702,10 @@ def test_collect_spot_check_source_paths_skips_directories():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')]
     )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
@@ -847,10 +847,10 @@ def test_collect_spot_check_source_paths_uses_working_directory():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
         flexmock()
     )
-    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')]
     )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(

tests/unit/actions/test_create.py

@@ -1,429 +1,9 @@
-import io
-import sys
+import os
 
 import pytest
 from flexmock import flexmock
 
 from borgmatic.actions import create as module
-from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
-
-
-@pytest.mark.parametrize(
-    'pattern_line,expected_pattern',
-    (
-        ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)),
-        ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)),
-        ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)),
-        (
-            '+ sh:/foo*',
-            Pattern(
-                '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG
-            ),
-        ),
-    ),
-)
-def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern):
-    module.parse_pattern(pattern_line) == expected_pattern
-
-
-def test_parse_pattern_with_invalid_pattern_line_errors():
-    with pytest.raises(ValueError):
-        module.parse_pattern('/foo')
-
-
-def test_collect_patterns_converts_source_directories():
-    assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == (
-        Pattern('/foo', source=Pattern_source.CONFIG),
-        Pattern('/bar', source=Pattern_source.CONFIG),
-    )
-
-
-def test_collect_patterns_parses_config_patterns():
-    flexmock(module).should_receive('parse_pattern').with_args('R /foo').and_return(Pattern('/foo'))
-    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
-    flexmock(module).should_receive('parse_pattern').with_args('').never()
-    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
-    flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
-
-    assert module.collect_patterns({'patterns': ['R /foo', '# comment', '', '   ', 'R /bar']}) == (
-        Pattern('/foo'),
-        Pattern('/bar'),
-    )
-
-
-def test_collect_patterns_converts_exclude_patterns():
-    assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
-        Pattern(
-            '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
-        ),
-        Pattern(
-            '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
-        ),
-        Pattern(
-            '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG
-        ),
-    )
-
-
-def test_collect_patterns_reads_config_patterns_from_file():
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('R /foo'))
-    builtins.should_receive('open').with_args('file2.txt').and_return(
-        io.StringIO('R /bar\n# comment\n\n   \nR /baz')
-    )
-    flexmock(module).should_receive('parse_pattern').with_args('R /foo').and_return(Pattern('/foo'))
-    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
-    flexmock(module).should_receive('parse_pattern').with_args('').never()
-    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
-    flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
-    flexmock(module).should_receive('parse_pattern').with_args('R /baz').and_return(Pattern('/baz'))
-
-    assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}) == (
-        Pattern('/foo'),
-        Pattern('/bar'),
-        Pattern('/baz'),
-    )
-
-
-def test_collect_patterns_errors_on_missing_config_patterns_from_file():
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').with_args('file1.txt').and_raise(FileNotFoundError)
-    flexmock(module).should_receive('parse_pattern').never()
-
-    with pytest.raises(ValueError):
-        module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']})
-
-
-def test_collect_patterns_reads_config_exclude_from_file():
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('/foo'))
-    builtins.should_receive('open').with_args('file2.txt').and_return(
-        io.StringIO('/bar\n# comment\n\n   \n/baz')
-    )
-    flexmock(module).should_receive('parse_pattern').with_args(
-        '! /foo', default_style=Pattern_style.FNMATCH
-    ).and_return(Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
-    flexmock(module).should_receive('parse_pattern').with_args(
-        '! /bar', default_style=Pattern_style.FNMATCH
-    ).and_return(Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
-    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
-    flexmock(module).should_receive('parse_pattern').with_args('').never()
-    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
-    flexmock(module).should_receive('parse_pattern').with_args(
-        '! /baz', default_style=Pattern_style.FNMATCH
-    ).and_return(Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
-
-    assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) == (
-        Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
-        Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
-        Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
-    )
-
-
-def test_collect_patterns_errors_on_missing_config_exclude_from_file():
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').with_args('file1.txt').and_raise(OSError)
-    flexmock(module).should_receive('parse_pattern').never()
-
-    with pytest.raises(ValueError):
-        module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']})
-
-
-def test_expand_directory_with_basic_path_passes_it_through():
-    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
-    flexmock(module.glob).should_receive('glob').and_return([])
-
-    paths = module.expand_directory('foo', None)
-
-    assert paths == ['foo']
-
-
-def test_expand_directory_with_glob_expands():
-    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
-    flexmock(module.glob).should_receive('glob').and_return(['foo', 'food'])
-
-    paths = module.expand_directory('foo*', None)
-
-    assert paths == ['foo', 'food']
-
-
-def test_expand_directory_strips_off_working_directory():
-    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
-    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo').and_return([]).once()
-
-    paths = module.expand_directory('foo', working_directory='/working/dir')
-
-    assert paths == ['foo']
-
-
-def test_expand_directory_globs_working_directory_and_strips_it_off():
-    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
-    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo*').and_return(
-        ['/working/dir/foo', '/working/dir/food']
-    ).once()
-
-    paths = module.expand_directory('foo*', working_directory='/working/dir')
-
-    assert paths == ['foo', 'food']
-
-
-def test_expand_directory_with_slashdot_hack_globs_working_directory_and_strips_it_off():
-    flexmock(module.os.path).should_receive('expanduser').and_return('./foo*')
-    flexmock(module.glob).should_receive('glob').with_args('/working/dir/./foo*').and_return(
-        ['/working/dir/./foo', '/working/dir/./food']
-    ).once()
-
-    paths = module.expand_directory('./foo*', working_directory='/working/dir')
-
-    assert paths == ['./foo', './food']
-
-
-def test_expand_directory_with_working_directory_matching_start_of_directory_does_not_strip_it_off():
-    flexmock(module.os.path).should_receive('expanduser').and_return('/working/dir/foo')
-    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo').and_return(
-        ['/working/dir/foo']
-    ).once()
-
-    paths = module.expand_directory('/working/dir/foo', working_directory='/working/dir')
-
-    assert paths == ['/working/dir/foo']
-
-
-def test_expand_patterns_flattens_expanded_directories():
-    flexmock(module).should_receive('expand_directory').with_args('~/foo', None).and_return(
-        ['/root/foo']
-    )
-    flexmock(module).should_receive('expand_directory').with_args('bar*', None).and_return(
-        ['bar', 'barf']
-    )
-
-    paths = module.expand_patterns((Pattern('~/foo'), Pattern('bar*')))
-
-    assert paths == (Pattern('/root/foo'), Pattern('bar'), Pattern('barf'))
-
-
-def test_expand_patterns_with_working_directory_passes_it_through():
-    flexmock(module).should_receive('expand_directory').with_args('foo', '/working/dir').and_return(
-        ['/working/dir/foo']
-    )
-
-    patterns = module.expand_patterns((Pattern('foo'),), working_directory='/working/dir')
-
-    assert patterns == (Pattern('/working/dir/foo'),)
-
-
-def test_expand_patterns_does_not_expand_skip_paths():
-    flexmock(module).should_receive('expand_directory').with_args('/foo', None).and_return(['/foo'])
-    flexmock(module).should_receive('expand_directory').with_args('/bar*', None).never()
-
-    patterns = module.expand_patterns((Pattern('/foo'), Pattern('/bar*')), skip_paths=('/bar*',))
-
-    assert patterns == (Pattern('/foo'), Pattern('/bar*'))
-
-
-def test_expand_patterns_considers_none_as_no_patterns():
-    assert module.expand_patterns(None) == ()
-
-
-def test_expand_patterns_expands_tildes_and_globs_in_root_patterns():
-    flexmock(module.os.path).should_receive('expanduser').never()
-    flexmock(module).should_receive('expand_directory').and_return(
-        ['/root/foo/one', '/root/foo/two']
-    )
-
-    paths = module.expand_patterns((Pattern('~/foo/*'),))
-
-    assert paths == (Pattern('/root/foo/one'), Pattern('/root/foo/two'))
-
-
-def test_expand_patterns_expands_only_tildes_in_non_root_patterns():
-    flexmock(module).should_receive('expand_directory').never()
-    flexmock(module.os.path).should_receive('expanduser').and_return('/root/bar/*')
-
-    paths = module.expand_patterns((Pattern('~/bar/*', Pattern_type.INCLUDE),))
-
-    assert paths == (Pattern('/root/bar/*', Pattern_type.INCLUDE),)
-
-
-def test_device_map_patterns_gives_device_id_per_path():
-    flexmock(module.os.path).should_receive('exists').and_return(True)
-    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
-    flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=66))
-
-    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar')))
-
-    assert device_map == (
-        Pattern('/foo', device=55),
-        Pattern('/bar', device=66),
-    )
-
-
-def test_device_map_patterns_only_considers_root_patterns():
-    flexmock(module.os.path).should_receive('exists').and_return(True)
-    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
-    flexmock(module.os).should_receive('stat').with_args('/bar*').never()
-
-    device_map = module.device_map_patterns(
-        (Pattern('/foo'), Pattern('/bar*', Pattern_type.INCLUDE))
-    )
-
-    assert device_map == (
-        Pattern('/foo', device=55),
-        Pattern('/bar*', Pattern_type.INCLUDE),
-    )
-
-
-def test_device_map_patterns_with_missing_path_does_not_error():
-    flexmock(module.os.path).should_receive('exists').and_return(True).and_return(False)
-    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
-    flexmock(module.os).should_receive('stat').with_args('/bar').never()
-
-    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar')))
-
-    assert device_map == (
-        Pattern('/foo', device=55),
-        Pattern('/bar'),
-    )
-
-
-def test_device_map_patterns_uses_working_directory_to_construct_path():
-    flexmock(module.os.path).should_receive('exists').and_return(True)
-    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
-    flexmock(module.os).should_receive('stat').with_args('/working/dir/bar').and_return(
-        flexmock(st_dev=66)
-    )
-
-    device_map = module.device_map_patterns(
-        (Pattern('/foo'), Pattern('bar')), working_directory='/working/dir'
-    )
-
-    assert device_map == (
-        Pattern('/foo', device=55),
-        Pattern('bar', device=66),
-    )
-
-
-def test_device_map_patterns_with_existing_device_id_does_not_overwrite_it():
-    flexmock(module.os.path).should_receive('exists').and_return(True)
-    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
-    flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=100))
-
-    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar', device=66)))
-
-    assert device_map == (
-        Pattern('/foo', device=55),
-        Pattern('/bar', device=66),
-    )
-
-
-@pytest.mark.parametrize(
-    'patterns,expected_patterns',
-    (
-        ((Pattern('/', device=1), Pattern('/root', device=1)), (Pattern('/', device=1),)),
-        ((Pattern('/', device=1), Pattern('/root/', device=1)), (Pattern('/', device=1),)),
-        (
-            (Pattern('/', device=1), Pattern('/root', device=2)),
-            (Pattern('/', device=1), Pattern('/root', device=2)),
-        ),
-        ((Pattern('/root', device=1), Pattern('/', device=1)), (Pattern('/', device=1),)),
-        (
-            (Pattern('/root', device=1), Pattern('/root/foo', device=1)),
-            (Pattern('/root', device=1),),
-        ),
-        (
-            (Pattern('/root/', device=1), Pattern('/root/foo', device=1)),
-            (Pattern('/root/', device=1),),
-        ),
-        (
-            (Pattern('/root', device=1), Pattern('/root/foo/', device=1)),
-            (Pattern('/root', device=1),),
-        ),
-        (
-            (Pattern('/root', device=1), Pattern('/root/foo', device=2)),
-            (Pattern('/root', device=1), Pattern('/root/foo', device=2)),
-        ),
-        (
-            (Pattern('/root/foo', device=1), Pattern('/root', device=1)),
-            (Pattern('/root', device=1),),
-        ),
-        (
-            (Pattern('/root', device=None), Pattern('/root/foo', device=None)),
-            (Pattern('/root'), Pattern('/root/foo')),
-        ),
-        (
-            (
-                Pattern('/root', device=1),
-                Pattern('/etc', device=1),
-                Pattern('/root/foo/bar', device=1),
-            ),
-            (Pattern('/root', device=1), Pattern('/etc', device=1)),
-        ),
-        (
-            (
-                Pattern('/root', device=1),
-                Pattern('/root/foo', device=1),
-                Pattern('/root/foo/bar', device=1),
-            ),
-            (Pattern('/root', device=1),),
-        ),
-        ((Pattern('/dup', device=1), Pattern('/dup', device=1)), (Pattern('/dup', device=1),)),
-        (
-            (Pattern('/foo', device=1), Pattern('/bar', device=1)),
-            (Pattern('/foo', device=1), Pattern('/bar', device=1)),
-        ),
-        (
-            (Pattern('/foo', device=1), Pattern('/bar', device=2)),
-            (Pattern('/foo', device=1), Pattern('/bar', device=2)),
-        ),
-        ((Pattern('/root/foo', device=1),), (Pattern('/root/foo', device=1),)),
-        (
-            (Pattern('/', device=1), Pattern('/root', Pattern_type.INCLUDE, device=1)),
-            (Pattern('/', device=1), Pattern('/root', Pattern_type.INCLUDE, device=1)),
-        ),
-        (
-            (Pattern('/root', Pattern_type.INCLUDE, device=1), Pattern('/', device=1)),
-            (Pattern('/root', Pattern_type.INCLUDE, device=1), Pattern('/', device=1)),
-        ),
-    ),
-)
-def test_deduplicate_patterns_omits_child_paths_on_the_same_filesystem(patterns, expected_patterns):
-    assert module.deduplicate_patterns(patterns) == expected_patterns
-
-
-def test_process_patterns_includes_patterns():
-    flexmock(module).should_receive('deduplicate_patterns').and_return(
-        (Pattern('foo'), Pattern('bar'))
-    )
-    flexmock(module).should_receive('device_map_patterns').and_return({})
-    flexmock(module).should_receive('expand_patterns').with_args(
-        (Pattern('foo'), Pattern('bar')),
-        working_directory='/working',
-        skip_paths=set(),
-    ).and_return(()).once()
-
-    assert module.process_patterns(
-        (Pattern('foo'), Pattern('bar')),
-        working_directory='/working',
-    ) == [Pattern('foo'), Pattern('bar')]
-
-
-def test_process_patterns_skips_expand_for_requested_paths():
-    skip_paths = {flexmock()}
-    flexmock(module).should_receive('deduplicate_patterns').and_return(
-        (Pattern('foo'), Pattern('bar'))
-    )
-    flexmock(module).should_receive('device_map_patterns').and_return({})
-    flexmock(module).should_receive('expand_patterns').with_args(
-        (Pattern('foo'), Pattern('bar')),
-        working_directory='/working',
-        skip_paths=skip_paths,
-    ).and_return(()).once()
-
-    assert module.process_patterns(
-        (Pattern('foo'), Pattern('bar')),
-        working_directory='/working',
-        skip_expand_paths=skip_paths,
-    ) == [Pattern('foo'), Pattern('bar')]
 
 
 def test_run_create_executes_and_calls_hooks_for_configured_repository():
@@ -437,9 +17,9 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
-    flexmock(module).should_receive('collect_patterns').and_return(())
-    flexmock(module).should_receive('process_patterns').and_return([])
-    flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(())
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return([])
+    flexmock(os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=None,
         progress=flexmock(),
@@ -478,9 +58,9 @@ def test_run_create_runs_with_selected_repository():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
-    flexmock(module).should_receive('collect_patterns').and_return(())
-    flexmock(module).should_receive('process_patterns').and_return([])
-    flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(())
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return([])
+    flexmock(os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),
@@ -621,9 +201,9 @@ def test_run_create_produces_json():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
-    flexmock(module).should_receive('collect_patterns').and_return(())
-    flexmock(module).should_receive('process_patterns').and_return([])
-    flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(())
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return([])
+    flexmock(os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),

tests/unit/actions/test_pattern.py

@@ -0,0 +1,459 @@
+import io
+import sys
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.actions import pattern as module
+from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
+
+
+@pytest.mark.parametrize(
+    'pattern_line,expected_pattern',
+    (
+        ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)),
+        ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)),
+        ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)),
+        (
+            '+ sh:/foo*',
+            Pattern(
+                '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG
+            ),
+        ),
+    ),
+)
+def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern):
+    module.parse_pattern(pattern_line) == expected_pattern
+
+
+def test_parse_pattern_with_invalid_pattern_line_errors():
+    with pytest.raises(ValueError):
+        module.parse_pattern('/foo')
+
+
+def test_collect_patterns_converts_source_directories():
+    assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == (
+        Pattern('/foo', source=Pattern_source.CONFIG),
+        Pattern('/bar', source=Pattern_source.CONFIG),
+    )
+
+
+def test_collect_patterns_parses_config_patterns():
+    flexmock(module).should_receive('parse_pattern').with_args('R /foo').and_return(Pattern('/foo'))
+    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
+
+    assert module.collect_patterns({'patterns': ['R /foo', '# comment', '', '   ', 'R /bar']}) == (
+        Pattern('/foo'),
+        Pattern('/bar'),
+    )
+
+
+def test_collect_patterns_converts_exclude_patterns():
+    assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
+        Pattern(
+            '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
+        ),
+        Pattern(
+            '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
+        ),
+        Pattern(
+            '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG
+        ),
+    )
+
+
+def test_collect_patterns_reads_config_patterns_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('R /foo'))
+    builtins.should_receive('open').with_args('file2.txt').and_return(
+        io.StringIO('R /bar\n# comment\n\n   \nR /baz')
+    )
+    flexmock(module).should_receive('parse_pattern').with_args('R /foo').and_return(Pattern('/foo'))
+    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
+    flexmock(module).should_receive('parse_pattern').with_args('R /baz').and_return(Pattern('/baz'))
+
+    assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}) == (
+        Pattern('/foo'),
+        Pattern('/bar'),
+        Pattern('/baz'),
+    )
+
+
+def test_collect_patterns_errors_on_missing_config_patterns_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_raise(FileNotFoundError)
+    flexmock(module).should_receive('parse_pattern').never()
+
+    with pytest.raises(ValueError):
+        module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']})
+
+
+def test_collect_patterns_reads_config_exclude_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('/foo'))
+    builtins.should_receive('open').with_args('file2.txt').and_return(
+        io.StringIO('/bar\n# comment\n\n   \n/baz')
+    )
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '! /foo', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '! /bar', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
+    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '! /baz', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
+
+    assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) == (
+        Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
+        Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
+        Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
+    )
+
+
+def test_collect_patterns_errors_on_missing_config_exclude_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_raise(OSError)
+    flexmock(module).should_receive('parse_pattern').never()
+
+    with pytest.raises(ValueError):
+        module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']})
+
+
+def test_expand_directory_with_basic_path_passes_it_through():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
+    flexmock(module.glob).should_receive('glob').and_return([])
+
+    paths = module.expand_directory('foo', None)
+
+    assert paths == ['foo']
+
+
+def test_expand_directory_with_glob_expands():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
+    flexmock(module.glob).should_receive('glob').and_return(['foo', 'food'])
+
+    paths = module.expand_directory('foo*', None)
+
+    assert paths == ['foo', 'food']
+
+
+def test_expand_directory_strips_off_working_directory():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo').and_return([]).once()
+
+    paths = module.expand_directory('foo', working_directory='/working/dir')
+
+    assert paths == ['foo']
+
+
+def test_expand_directory_globs_working_directory_and_strips_it_off():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo*').and_return(
+        ['/working/dir/foo', '/working/dir/food']
+    ).once()
+
+    paths = module.expand_directory('foo*', working_directory='/working/dir')
+
+    assert paths == ['foo', 'food']
+
+
+def test_expand_directory_with_slashdot_hack_globs_working_directory_and_strips_it_off():
+    flexmock(module.os.path).should_receive('expanduser').and_return('./foo*')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/./foo*').and_return(
+        ['/working/dir/./foo', '/working/dir/./food']
+    ).once()
+
+    paths = module.expand_directory('./foo*', working_directory='/working/dir')
+
+    assert paths == ['./foo', './food']
+
+
+def test_expand_directory_with_working_directory_matching_start_of_directory_does_not_strip_it_off():
+    flexmock(module.os.path).should_receive('expanduser').and_return('/working/dir/foo')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo').and_return(
+        ['/working/dir/foo']
+    ).once()
+
+    paths = module.expand_directory('/working/dir/foo', working_directory='/working/dir')
+
+    assert paths == ['/working/dir/foo']
+
+
+def test_expand_patterns_flattens_expanded_directories():
+    flexmock(module).should_receive('expand_directory').with_args('~/foo', None).and_return(
+        ['/root/foo']
+    )
+    flexmock(module).should_receive('expand_directory').with_args('bar*', None).and_return(
+        ['bar', 'barf']
+    )
+
+    paths = module.expand_patterns((Pattern('~/foo'), Pattern('bar*')))
+
+    assert paths == (Pattern('/root/foo'), Pattern('bar'), Pattern('barf'))
+
+
+def test_expand_patterns_with_working_directory_passes_it_through():
+    flexmock(module).should_receive('expand_directory').with_args('foo', '/working/dir').and_return(
+        ['/working/dir/foo']
+    )
+
+    patterns = module.expand_patterns((Pattern('foo'),), working_directory='/working/dir')
+
+    assert patterns == (Pattern('/working/dir/foo'),)
+
+
+def test_expand_patterns_does_not_expand_skip_paths():
+    flexmock(module).should_receive('expand_directory').with_args('/foo', None).and_return(['/foo'])
+    flexmock(module).should_receive('expand_directory').with_args('/bar*', None).never()
+
+    patterns = module.expand_patterns((Pattern('/foo'), Pattern('/bar*')), skip_paths=('/bar*',))
+
+    assert patterns == (Pattern('/foo'), Pattern('/bar*'))
+
+
+def test_expand_patterns_considers_none_as_no_patterns():
+    assert module.expand_patterns(None) == ()
+
+
+def test_expand_patterns_expands_tildes_and_globs_in_root_patterns():
+    flexmock(module.os.path).should_receive('expanduser').never()
+    flexmock(module).should_receive('expand_directory').and_return(
+        ['/root/foo/one', '/root/foo/two']
+    )
+
+    paths = module.expand_patterns((Pattern('~/foo/*'),))
+
+    assert paths == (Pattern('/root/foo/one'), Pattern('/root/foo/two'))
+
+
+def test_expand_patterns_expands_only_tildes_in_non_root_patterns():
+    flexmock(module).should_receive('expand_directory').never()
+    flexmock(module.os.path).should_receive('expanduser').and_return('/root/bar/*')
+
+    paths = module.expand_patterns((Pattern('~/bar/*', Pattern_type.INCLUDE),))
+
+    assert paths == (Pattern('/root/bar/*', Pattern_type.INCLUDE),)
+
+
+def test_get_existent_path_or_parent_passes_through_existent_path():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+
+    assert module.get_existent_path_or_parent('/foo/bar/baz') == '/foo/bar/baz'
+
+
+def test_get_existent_path_or_parent_with_non_existent_path_returns_none():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+
+    assert module.get_existent_path_or_parent('/foo/bar/baz') is None
+
+
+def test_get_existent_path_or_parent_with_non_existent_path_returns_existent_parent():
+    flexmock(module.os.path).should_receive('exists').with_args('/foo/bar/baz*').and_return(False)
+    flexmock(module.os.path).should_receive('exists').with_args('/foo/bar').and_return(True)
+    flexmock(module.os.path).should_receive('exists').with_args('/foo').never()
+    flexmock(module.os.path).should_receive('exists').with_args('/').never()
+
+    assert module.get_existent_path_or_parent('/foo/bar/baz*') == '/foo/bar'
+
+
+def test_get_existent_path_or_parent_with_non_existent_path_returns_existent_grandparent():
+    flexmock(module.os.path).should_receive('exists').with_args('/foo/bar/baz*').and_return(False)
+    flexmock(module.os.path).should_receive('exists').with_args('/foo/bar').and_return(False)
+    flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True)
+    flexmock(module.os.path).should_receive('exists').with_args('/').never()
+
+    assert module.get_existent_path_or_parent('/foo/bar/baz*') == '/foo'
+
+
+def test_get_existent_path_or_parent_with_end_to_end_test_prefix_returns_none():
+    flexmock(module.os.path).should_receive('exists').never()
+
+    assert module.get_existent_path_or_parent('/e2e/foo/bar/baz') is None
+
+
+def test_device_map_patterns_gives_device_id_per_path():
+    flexmock(module).should_receive('get_existent_path_or_parent').replace_with(lambda path: path)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=66))
+
+    device_map = module.device_map_patterns(
+        (
+            Pattern('/foo'),
+            Pattern('^/bar', type=Pattern_type.INCLUDE, style=Pattern_style.REGULAR_EXPRESSION),
+        )
+    )
+
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern(
+            '^/bar', type=Pattern_type.INCLUDE, style=Pattern_style.REGULAR_EXPRESSION, device=66
+        ),
+    )
+
+
+def test_device_map_patterns_with_missing_path_does_not_error():
+    flexmock(module).should_receive('get_existent_path_or_parent').with_args('/foo').and_return(
+        '/foo'
+    )
+    flexmock(module).should_receive('get_existent_path_or_parent').with_args('/bar').and_return(
+        None
+    )
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').never()
+
+    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar')))
+
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('/bar'),
+    )
+
+
+def test_device_map_patterns_uses_working_directory_to_construct_path():
+    flexmock(module).should_receive('get_existent_path_or_parent').replace_with(lambda path: path)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/working/dir/bar').and_return(
+        flexmock(st_dev=66)
+    )
+
+    device_map = module.device_map_patterns(
+        (Pattern('/foo'), Pattern('bar')), working_directory='/working/dir'
+    )
+
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('bar', device=66),
+    )
+
+
+def test_device_map_patterns_with_existing_device_id_does_not_overwrite_it():
+    flexmock(module).should_receive('get_existent_path_or_parent').replace_with(lambda path: path)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=100))
+
+    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar', device=66)))
+
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('/bar', device=66),
+    )
+
+
+@pytest.mark.parametrize(
+    'patterns,expected_patterns',
+    (
+        ((Pattern('/', device=1), Pattern('/root', device=1)), (Pattern('/', device=1),)),
+        ((Pattern('/', device=1), Pattern('/root/', device=1)), (Pattern('/', device=1),)),
+        (
+            (Pattern('/', device=1), Pattern('/root', device=2)),
+            (Pattern('/', device=1), Pattern('/root', device=2)),
+        ),
+        ((Pattern('/root', device=1), Pattern('/', device=1)), (Pattern('/', device=1),)),
+        (
+            (Pattern('/root', device=1), Pattern('/root/foo', device=1)),
+            (Pattern('/root', device=1),),
+        ),
+        (
+            (Pattern('/root/', device=1), Pattern('/root/foo', device=1)),
+            (Pattern('/root/', device=1),),
+        ),
+        (
+            (Pattern('/root', device=1), Pattern('/root/foo/', device=1)),
+            (Pattern('/root', device=1),),
+        ),
+        (
+            (Pattern('/root', device=1), Pattern('/root/foo', device=2)),
+            (Pattern('/root', device=1), Pattern('/root/foo', device=2)),
+        ),
+        (
+            (Pattern('/root/foo', device=1), Pattern('/root', device=1)),
+            (Pattern('/root', device=1),),
+        ),
+        (
+            (Pattern('/root', device=None), Pattern('/root/foo', device=None)),
+            (Pattern('/root'), Pattern('/root/foo')),
+        ),
+        (
+            (
+                Pattern('/root', device=1),
+                Pattern('/etc', device=1),
+                Pattern('/root/foo/bar', device=1),
+            ),
+            (Pattern('/root', device=1), Pattern('/etc', device=1)),
+        ),
+        (
+            (
+                Pattern('/root', device=1),
+                Pattern('/root/foo', device=1),
+                Pattern('/root/foo/bar', device=1),
+            ),
+            (Pattern('/root', device=1),),
+        ),
+        ((Pattern('/dup', device=1), Pattern('/dup', device=1)), (Pattern('/dup', device=1),)),
+        (
+            (Pattern('/foo', device=1), Pattern('/bar', device=1)),
+            (Pattern('/foo', device=1), Pattern('/bar', device=1)),
+        ),
+        (
+            (Pattern('/foo', device=1), Pattern('/bar', device=2)),
+            (Pattern('/foo', device=1), Pattern('/bar', device=2)),
+        ),
+        ((Pattern('/root/foo', device=1),), (Pattern('/root/foo', device=1),)),
+        (
+            (Pattern('/', device=1), Pattern('/root', Pattern_type.INCLUDE, device=1)),
+            (Pattern('/', device=1), Pattern('/root', Pattern_type.INCLUDE, device=1)),
+        ),
+        (
+            (Pattern('/root', Pattern_type.INCLUDE, device=1), Pattern('/', device=1)),
+            (Pattern('/root', Pattern_type.INCLUDE, device=1), Pattern('/', device=1)),
+        ),
+    ),
+)
+def test_deduplicate_patterns_omits_child_paths_on_the_same_filesystem(patterns, expected_patterns):
+    assert module.deduplicate_patterns(patterns) == expected_patterns
+
+
+def test_process_patterns_includes_patterns():
+    flexmock(module).should_receive('deduplicate_patterns').and_return(
+        (Pattern('foo'), Pattern('bar'))
+    )
+    flexmock(module).should_receive('device_map_patterns').and_return({})
+    flexmock(module).should_receive('expand_patterns').with_args(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+        skip_paths=set(),
+    ).and_return(()).once()
+
+    assert module.process_patterns(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+    ) == [Pattern('foo'), Pattern('bar')]
+
+
+def test_process_patterns_skips_expand_for_requested_paths():
+    skip_paths = {flexmock()}
+    flexmock(module).should_receive('deduplicate_patterns').and_return(
+        (Pattern('foo'), Pattern('bar'))
+    )
+    flexmock(module).should_receive('device_map_patterns').and_return({})
+    flexmock(module).should_receive('expand_patterns').with_args(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+        skip_paths=skip_paths,
+    ).and_return(()).once()
+
+    assert module.process_patterns(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+        skip_expand_paths=skip_paths,
+    ) == [Pattern('foo'), Pattern('bar')]

tests/unit/actions/test_recreate.py

@@ -1,3 +1,4 @@
+import pytest
 from flexmock import flexmock
 
 from borgmatic.actions import recreate as module
@@ -6,15 +7,19 @@ from borgmatic.actions import recreate as module
 def test_run_recreate_does_not_raise():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        None
+    )
     flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive')
 
-    recreate_arguments = flexmock(repository=flexmock(), archive=None)
-
     module.run_recreate(
         repository={'path': 'repo'},
         config={},
         local_borg_version=None,
-        recreate_arguments=recreate_arguments,
+        recreate_arguments=flexmock(repository=flexmock(), archive=None),
         global_arguments=flexmock(),
         local_path=None,
         remote_path=None,
@@ -24,16 +29,154 @@ def test_run_recreate_does_not_raise():
 def test_run_recreate_with_archive_does_not_raise():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'test-archive'
+    )
     flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive')
 
-    recreate_arguments = flexmock(repository=flexmock(), archive='test-archive')
-
     module.run_recreate(
         repository={'path': 'repo'},
         config={},
         local_borg_version=None,
-        recreate_arguments=recreate_arguments,
+        recreate_arguments=flexmock(repository=flexmock(), archive='test-archive'),
         global_arguments=flexmock(),
         local_path=None,
         remote_path=None,
     )
+
+
+def test_run_recreate_with_leftover_recreate_archive_raises():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'test-archive.recreate'
+    )
+    flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive')
+
+    with pytest.raises(ValueError):
+        module.run_recreate(
+            repository={'path': 'repo'},
+            config={},
+            local_borg_version=None,
+            recreate_arguments=flexmock(repository=flexmock(), archive='test-archive.recreate'),
+            global_arguments=flexmock(),
+            local_path=None,
+            remote_path=None,
+        )
+
+
+def test_run_recreate_with_latest_archive_resolving_to_leftover_recreate_archive_raises():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'test-archive.recreate'
+    )
+    flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive')
+
+    with pytest.raises(ValueError):
+        module.run_recreate(
+            repository={'path': 'repo'},
+            config={},
+            local_borg_version=None,
+            recreate_arguments=flexmock(repository=flexmock(), archive='latest'),
+            global_arguments=flexmock(),
+            local_path=None,
+            remote_path=None,
+        )
+
+
+def test_run_recreate_with_archive_already_exists_error_raises():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'test-archive'
+    )
+    flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive').and_raise(
+        module.subprocess.CalledProcessError(
+            returncode=module.BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS,
+            cmd='borg recreate or whatever',
+        )
+    )
+
+    with pytest.raises(ValueError):
+        module.run_recreate(
+            repository={'path': 'repo'},
+            config={},
+            local_borg_version=None,
+            recreate_arguments=flexmock(repository=flexmock(), archive='test-archive', target=None),
+            global_arguments=flexmock(),
+            local_path=None,
+            remote_path=None,
+        )
+
+
+def test_run_recreate_with_target_and_archive_already_exists_error_raises():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'test-archive'
+    )
+    flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive').and_raise(
+        module.subprocess.CalledProcessError(
+            returncode=module.BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS,
+            cmd='borg recreate or whatever',
+        )
+    )
+
+    with pytest.raises(ValueError):
+        module.run_recreate(
+            repository={'path': 'repo'},
+            config={},
+            local_borg_version=None,
+            recreate_arguments=flexmock(
+                repository=flexmock(), archive='test-archive', target='target-archive'
+            ),
+            global_arguments=flexmock(),
+            local_path=None,
+            remote_path=None,
+        )
+
+
+def test_run_recreate_with_other_called_process_error_passes_it_through():
+    flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'test-archive'
+    )
+    flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive').and_raise(
+        module.subprocess.CalledProcessError(
+            returncode=1,
+            cmd='borg recreate or whatever',
+        )
+    )
+
+    with pytest.raises(module.subprocess.CalledProcessError):
+        module.run_recreate(
+            repository={'path': 'repo'},
+            config={},
+            local_borg_version=None,
+            recreate_arguments=flexmock(
+                repository=flexmock(), archive='test-archive', target='target-archive'
+            ),
+            global_arguments=flexmock(),
+            local_path=None,
+            remote_path=None,
+        )

tests/unit/borg/test_create.py

@@ -9,131 +9,6 @@ from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Patte
 from ..test_verbosity import insert_logging_mock
 
 
-def test_write_patterns_file_writes_pattern_lines():
-    temporary_file = flexmock(name='filename', flush=lambda: None)
-    temporary_file.should_receive('write').with_args('R /foo\n+ sh:/foo/bar')
-    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
-
-    module.write_patterns_file(
-        [Pattern('/foo'), Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.SHELL)],
-        borgmatic_runtime_directory='/run/user/0',
-    )
-
-
-def test_write_patterns_file_with_empty_exclude_patterns_does_not_raise():
-    module.write_patterns_file([], borgmatic_runtime_directory='/run/user/0')
-
-
-def test_write_patterns_file_appends_to_existing():
-    patterns_file = flexmock(name='filename', flush=lambda: None)
-    patterns_file.should_receive('write').with_args('\n')
-    patterns_file.should_receive('write').with_args('R /foo\n+ /foo/bar')
-    flexmock(module.tempfile).should_receive('NamedTemporaryFile').never()
-
-    module.write_patterns_file(
-        [Pattern('/foo'), Pattern('/foo/bar', Pattern_type.INCLUDE)],
-        borgmatic_runtime_directory='/run/user/0',
-        patterns_file=patterns_file,
-    )
-
-
-def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
-    exclude_flags = module.make_exclude_flags(config={'exclude_caches': True})
-
-    assert exclude_flags == ('--exclude-caches',)
-
-
-def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
-    exclude_flags = module.make_exclude_flags(config={'exclude_caches': False})
-
-    assert exclude_flags == ()
-
-
-def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
-    exclude_flags = module.make_exclude_flags(
-        config={'exclude_if_present': ['exclude_me', 'also_me']}
-    )
-
-    assert exclude_flags == (
-        '--exclude-if-present',
-        'exclude_me',
-        '--exclude-if-present',
-        'also_me',
-    )
-
-
-def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config():
-    exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': True})
-
-    assert exclude_flags == ('--keep-exclude-tags',)
-
-
-def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config():
-    exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': False})
-
-    assert exclude_flags == ()
-
-
-def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config():
-    exclude_flags = module.make_exclude_flags(config={'exclude_nodump': True})
-
-    assert exclude_flags == ('--exclude-nodump',)
-
-
-def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config():
-    exclude_flags = module.make_exclude_flags(config={'exclude_nodump': False})
-
-    assert exclude_flags == ()
-
-
-def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
-    exclude_flags = module.make_exclude_flags(config={})
-
-    assert exclude_flags == ()
-
-
-def test_make_list_filter_flags_with_debug_and_feature_available_includes_plus_and_minus():
-    flexmock(module.logger).should_receive('isEnabledFor').and_return(True)
-    flexmock(module.feature).should_receive('available').and_return(True)
-
-    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME+-'
-
-
-def test_make_list_filter_flags_with_info_and_feature_available_omits_plus_and_minus():
-    flexmock(module.logger).should_receive('isEnabledFor').and_return(False)
-    flexmock(module.feature).should_receive('available').and_return(True)
-
-    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME'
-
-
-def test_make_list_filter_flags_with_debug_and_feature_available_and_dry_run_includes_plus_and_minus():
-    flexmock(module.logger).should_receive('isEnabledFor').and_return(True)
-    flexmock(module.feature).should_receive('available').and_return(True)
-
-    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=True) == 'AME+-'
-
-
-def test_make_list_filter_flags_with_info_and_feature_available_and_dry_run_includes_plus_and_minus():
-    flexmock(module.logger).should_receive('isEnabledFor').and_return(False)
-    flexmock(module.feature).should_receive('available').and_return(True)
-
-    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=True) == 'AME+-'
-
-
-def test_make_list_filter_flags_with_debug_and_feature_not_available_includes_x():
-    flexmock(module.logger).should_receive('isEnabledFor').and_return(True)
-    flexmock(module.feature).should_receive('available').and_return(False)
-
-    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AMEx-'
-
-
-def test_make_list_filter_flags_with_info_and_feature_not_available_omits_x():
-    flexmock(module.logger).should_receive('isEnabledFor').and_return(False)
-    flexmock(module.feature).should_receive('available').and_return(False)
-
-    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME-'
-
-
 @pytest.mark.parametrize(
     'character_device,block_device,fifo,expected_result',
     (
@@ -326,10 +201,10 @@ REPO_ARCHIVE = (f'repo::{DEFAULT_ARCHIVE_NAME}',)
 
 def test_make_base_create_produces_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -355,16 +230,16 @@ def test_make_base_create_produces_borg_command():
 def test_make_base_create_command_includes_patterns_file_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
     mock_pattern_file = flexmock(name='/tmp/patterns')
-    flexmock(module).should_receive('write_patterns_file').and_return(mock_pattern_file).and_return(
-        None
-    )
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(
+        mock_pattern_file
+    ).and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
     pattern_flags = ('--patterns-from', mock_pattern_file.name)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -390,13 +265,13 @@ def test_make_base_create_command_includes_patterns_file_in_borg_command():
 
 def test_make_base_create_command_with_store_config_false_omits_config_files():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -455,13 +330,13 @@ def test_make_base_create_command_includes_configuration_option_as_command_flag(
     option_name, option_value, feature_available, option_flags
 ):
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(feature_available)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -487,13 +362,13 @@ def test_make_base_create_command_includes_configuration_option_as_command_flag(
 
 def test_make_base_create_command_includes_dry_run_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -519,13 +394,13 @@ def test_make_base_create_command_includes_dry_run_in_borg_command():
 
 def test_make_base_create_command_includes_local_path_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -551,13 +426,13 @@ def test_make_base_create_command_includes_local_path_in_borg_command():
 
 def test_make_base_create_command_includes_remote_path_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -583,13 +458,13 @@ def test_make_base_create_command_includes_remote_path_in_borg_command():
 
 def test_make_base_create_command_includes_log_json_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -615,13 +490,13 @@ def test_make_base_create_command_includes_log_json_in_borg_command():
 
 def test_make_base_create_command_includes_list_flags_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -649,22 +524,22 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
     patterns = [Pattern('foo'), Pattern('bar')]
     patterns_file = flexmock(name='patterns')
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').with_args(
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
         patterns, '/run/borgmatic'
     ).and_return(patterns_file)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
     flexmock(module.logger).should_receive('warning').twice()
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('collect_special_file_paths').and_return(('/dev/null',)).once()
-    flexmock(module).should_receive('write_patterns_file').with_args(
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
         (
             Pattern(
                 '/dev/null',
@@ -676,7 +551,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
         '/run/borgmatic',
         patterns_file=patterns_file,
     ).and_return(patterns_file).once()
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
 
     (create_flags, create_positional_arguments, pattern_file) = module.make_base_create_command(
         dry_run=False,
@@ -700,22 +575,22 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
 
 def test_make_base_create_command_without_patterns_and_with_stream_processes_ignores_read_special_false_and_excludes_special_files():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').with_args(
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
         [], '/run/borgmatic'
     ).and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
     flexmock(module.logger).should_receive('warning').twice()
     flexmock(module.environment).should_receive('make_environment')
     flexmock(module).should_receive('collect_special_file_paths').and_return(('/dev/null',)).once()
-    flexmock(module).should_receive('write_patterns_file').with_args(
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
         (
             Pattern(
                 '/dev/null',
@@ -727,7 +602,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
         '/run/borgmatic',
         patterns_file=None,
     ).and_return(flexmock(name='patterns')).once()
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
 
     (create_flags, create_positional_arguments, pattern_file) = module.make_base_create_command(
         dry_run=False,
@@ -751,13 +626,13 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
 
 def test_make_base_create_command_with_stream_processes_and_read_special_true_skips_special_files_excludes():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -786,13 +661,13 @@ def test_make_base_create_command_with_stream_processes_and_read_special_true_sk
 
 def test_make_base_create_command_includes_archive_name_format_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         ('repo::ARCHIVE_NAME',)
     )
@@ -818,13 +693,13 @@ def test_make_base_create_command_includes_archive_name_format_in_borg_command()
 
 def test_make_base_create_command_includes_default_archive_name_format_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         ('repo::{hostname}',)
     )
@@ -849,13 +724,13 @@ def test_make_base_create_command_includes_default_archive_name_format_in_borg_c
 
 def test_make_base_create_command_includes_archive_name_format_with_placeholders_in_borg_command():
     repository_archive_pattern = 'repo::Documents_{hostname}-{now}'  # noqa: FS003
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (repository_archive_pattern,)
     )
@@ -881,13 +756,13 @@ def test_make_base_create_command_includes_archive_name_format_with_placeholders
 
 def test_make_base_create_command_includes_repository_and_archive_name_format_with_placeholders_in_borg_command():
     repository_archive_pattern = '{fqdn}::Documents_{hostname}-{now}'  # noqa: FS003
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (repository_archive_pattern,)
     )
@@ -913,13 +788,13 @@ def test_make_base_create_command_includes_repository_and_archive_name_format_wi
 
 def test_make_base_create_command_includes_extra_borg_options_in_borg_command():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('write_patterns_file').and_return(None)
-    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('FOO')
     flexmock(module.flags).should_receive('get_default_archive_name_format').and_return(
         '{hostname}'
     )
     flexmock(module.feature).should_receive('available').and_return(True)
-    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
     flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
@@ -945,7 +820,9 @@ def test_make_base_create_command_includes_extra_borg_options_in_borg_command():
 
 def test_make_base_create_command_with_non_existent_directory_and_source_directories_must_exist_raises():
     flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
-    flexmock(module).should_receive('check_all_root_patterns_exist').and_raise(ValueError)
+    flexmock(module.borgmatic.borg.pattern).should_receive(
+        'check_all_root_patterns_exist'
+    ).and_raise(ValueError)
 
     with pytest.raises(ValueError):
         module.make_base_create_command(
@@ -1571,22 +1448,3 @@ def test_create_archive_calls_borg_with_working_directory():
         global_arguments=flexmock(),
         borgmatic_runtime_directory='/borgmatic/run',
     )
-
-
-def test_check_all_root_patterns_exist_with_existent_pattern_path_does_not_raise():
-    flexmock(module.os.path).should_receive('exists').and_return(True)
-
-    module.check_all_root_patterns_exist([Pattern('foo')])
-
-
-def test_check_all_root_patterns_exist_with_non_root_pattern_skips_existence_check():
-    flexmock(module.os.path).should_receive('exists').never()
-
-    module.check_all_root_patterns_exist([Pattern('foo', Pattern_type.INCLUDE)])
-
-
-def test_check_all_root_patterns_exist_with_non_existent_pattern_path_raises():
-    flexmock(module.os.path).should_receive('exists').and_return(False)
-
-    with pytest.raises(ValueError):
-        module.check_all_root_patterns_exist([Pattern('foo')])

tests/unit/borg/test_flags.py

@@ -327,3 +327,100 @@ def test_omit_flag_and_value_without_flag_present_passes_through_arguments():
         'create',
         '--other',
     )
+
+
+def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
+    exclude_flags = module.make_exclude_flags(config={'exclude_caches': True})
+
+    assert exclude_flags == ('--exclude-caches',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
+    exclude_flags = module.make_exclude_flags(config={'exclude_caches': False})
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
+    exclude_flags = module.make_exclude_flags(
+        config={'exclude_if_present': ['exclude_me', 'also_me']}
+    )
+
+    assert exclude_flags == (
+        '--exclude-if-present',
+        'exclude_me',
+        '--exclude-if-present',
+        'also_me',
+    )
+
+
+def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config():
+    exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': True})
+
+    assert exclude_flags == ('--keep-exclude-tags',)
+
+
+def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config():
+    exclude_flags = module.make_exclude_flags(config={'keep_exclude_tags': False})
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config():
+    exclude_flags = module.make_exclude_flags(config={'exclude_nodump': True})
+
+    assert exclude_flags == ('--exclude-nodump',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config():
+    exclude_flags = module.make_exclude_flags(config={'exclude_nodump': False})
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
+    exclude_flags = module.make_exclude_flags(config={})
+
+    assert exclude_flags == ()
+
+
+def test_make_list_filter_flags_with_debug_and_feature_available_includes_plus_and_minus():
+    flexmock(module.logger).should_receive('isEnabledFor').and_return(True)
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME+-'
+
+
+def test_make_list_filter_flags_with_info_and_feature_available_omits_plus_and_minus():
+    flexmock(module.logger).should_receive('isEnabledFor').and_return(False)
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME'
+
+
+def test_make_list_filter_flags_with_debug_and_feature_available_and_dry_run_includes_plus_and_minus():
+    flexmock(module.logger).should_receive('isEnabledFor').and_return(True)
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=True) == 'AME+-'
+
+
+def test_make_list_filter_flags_with_info_and_feature_available_and_dry_run_includes_plus_and_minus():
+    flexmock(module.logger).should_receive('isEnabledFor').and_return(False)
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=True) == 'AME+-'
+
+
+def test_make_list_filter_flags_with_debug_and_feature_not_available_includes_x():
+    flexmock(module.logger).should_receive('isEnabledFor').and_return(True)
+    flexmock(module.feature).should_receive('available').and_return(False)
+
+    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AMEx-'
+
+
+def test_make_list_filter_flags_with_info_and_feature_not_available_omits_x():
+    flexmock(module.logger).should_receive('isEnabledFor').and_return(False)
+    flexmock(module.feature).should_receive('available').and_return(False)
+
+    assert module.make_list_filter_flags(local_borg_version=flexmock(), dry_run=False) == 'AME-'

tests/unit/borg/test_pattern.py

@@ -0,0 +1,52 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.borg import pattern as module
+from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
+
+
+def test_write_patterns_file_writes_pattern_lines():
+    temporary_file = flexmock(name='filename', flush=lambda: None)
+    temporary_file.should_receive('write').with_args('R /foo\n+ sh:/foo/bar')
+    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
+
+    module.write_patterns_file(
+        [Pattern('/foo'), Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.SHELL)],
+        borgmatic_runtime_directory='/run/user/0',
+    )
+
+
+def test_write_patterns_file_with_empty_exclude_patterns_does_not_raise():
+    module.write_patterns_file([], borgmatic_runtime_directory='/run/user/0')
+
+
+def test_write_patterns_file_appends_to_existing():
+    patterns_file = flexmock(name='filename', flush=lambda: None)
+    patterns_file.should_receive('write').with_args('\n')
+    patterns_file.should_receive('write').with_args('R /foo\n+ /foo/bar')
+    flexmock(module.tempfile).should_receive('NamedTemporaryFile').never()
+
+    module.write_patterns_file(
+        [Pattern('/foo'), Pattern('/foo/bar', Pattern_type.INCLUDE)],
+        borgmatic_runtime_directory='/run/user/0',
+        patterns_file=patterns_file,
+    )
+
+
+def test_check_all_root_patterns_exist_with_existent_pattern_path_does_not_raise():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+
+    module.check_all_root_patterns_exist([Pattern('foo')])
+
+
+def test_check_all_root_patterns_exist_with_non_root_pattern_skips_existence_check():
+    flexmock(module.os.path).should_receive('exists').never()
+
+    module.check_all_root_patterns_exist([Pattern('foo', Pattern_type.INCLUDE)])
+
+
+def test_check_all_root_patterns_exist_with_non_existent_pattern_path_raises():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+
+    with pytest.raises(ValueError):
+        module.check_all_root_patterns_exist([Pattern('foo')])

tests/unit/borg/test_recreate.py

@@ -21,9 +21,9 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes
 
 
 def test_recreate_archive_dry_run_skips_execution():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -59,9 +59,9 @@ def test_recreate_archive_dry_run_skips_execution():
 
 
 def test_recreate_calls_borg_with_required_flags():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -94,9 +94,9 @@ def test_recreate_calls_borg_with_required_flags():
 
 
 def test_recreate_with_remote_path():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -129,9 +129,9 @@ def test_recreate_with_remote_path():
 
 
 def test_recreate_with_lock_wait():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -163,9 +163,9 @@ def test_recreate_with_lock_wait():
 
 
 def test_recreate_with_log_info():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -199,9 +199,9 @@ def test_recreate_with_log_info():
 
 
 def test_recreate_with_log_debug():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -234,9 +234,9 @@ def test_recreate_with_log_debug():
 
 
 def test_recreate_with_log_json():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -268,8 +268,8 @@ def test_recreate_with_log_json():
 
 
 def test_recreate_with_list_config_calls_borg_with_list_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -280,7 +280,9 @@ def test_recreate_with_list_config_calls_borg_with_list_flag():
             'repo',
         )
     )
-    flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return(
+        'AME+-'
+    )
     insert_execute_command_mock(
         ('borg', 'recreate', '--list', '--filter', 'AME+-', '--repo', 'repo')
     )
@@ -304,8 +306,8 @@ def test_recreate_with_list_config_calls_borg_with_list_flag():
 
 
 def test_recreate_with_patterns_from_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -341,8 +343,8 @@ def test_recreate_with_patterns_from_flag():
 
 
 def test_recreate_with_exclude_flags():
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -353,7 +355,9 @@ def test_recreate_with_exclude_flags():
             'repo',
         )
     )
-    flexmock(module).should_receive('make_exclude_flags').and_return(('--exclude', 'pattern'))
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(
+        ('--exclude', 'pattern')
+    )
     insert_execute_command_mock(('borg', 'recreate', '--exclude', 'pattern', '--repo', 'repo'))
 
     module.recreate_archive(
@@ -375,9 +379,9 @@ def test_recreate_with_exclude_flags():
 
 
 def test_recreate_with_target_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -409,9 +413,9 @@ def test_recreate_with_target_flag():
 
 
 def test_recreate_with_comment_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -445,9 +449,9 @@ def test_recreate_with_comment_flag():
 
 
 def test_recreate_with_timestamp_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -481,9 +485,9 @@ def test_recreate_with_timestamp_flag():
 
 
 def test_recreate_with_compression_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -515,9 +519,9 @@ def test_recreate_with_compression_flag():
 
 
 def test_recreate_with_chunker_params_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -551,9 +555,9 @@ def test_recreate_with_chunker_params_flag():
 
 
 def test_recreate_with_recompress_flag():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -585,9 +589,9 @@ def test_recreate_with_recompress_flag():
 
 
 def test_recreate_with_match_archives_star():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -619,9 +623,9 @@ def test_recreate_with_match_archives_star():
 
 
 def test_recreate_with_match_archives_regex():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -653,9 +657,9 @@ def test_recreate_with_match_archives_regex():
 
 
 def test_recreate_with_match_archives_shell():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(())
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True)
     flexmock(module.borgmatic.borg.flags).should_receive(
@@ -687,9 +691,9 @@ def test_recreate_with_match_archives_shell():
 
 
 def test_recreate_with_match_archives_and_feature_available_calls_borg_with_match_archives():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').with_args(
         'foo-*', None, '1.2.3'
     ).and_return(('--match-archives', 'foo-*'))
@@ -719,9 +723,9 @@ def test_recreate_with_match_archives_and_feature_available_calls_borg_with_matc
 
 
 def test_recreate_with_archives_flag_and_feature_available_calls_borg_with_match_archives():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').with_args(
         'archive', None, '1.2.3'
     ).and_return(('--match-archives', 'archive'))
@@ -753,9 +757,9 @@ def test_recreate_with_archives_flag_and_feature_available_calls_borg_with_match
 
 
 def test_recreate_with_match_archives_and_feature_not_available_calls_borg_without_match_archives():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').never()
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(False)
     flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return(
@@ -783,9 +787,9 @@ def test_recreate_with_match_archives_and_feature_not_available_calls_borg_witho
 
 
 def test_recreate_with_archives_flags_and_feature_not_available_calls_borg_with_combined_repo_and_archive():
-    flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(())
-    flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None)
-    flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('')
+    flexmock(module.borgmatic.borg.flags).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').and_return(None)
+    flexmock(module.borgmatic.borg.flags).should_receive('make_list_filter_flags').and_return('')
     flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').never()
     flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(False)
     flexmock(module.borgmatic.borg.flags).should_receive(

tests/unit/commands/test_borgmatic.py

@@ -465,8 +465,10 @@ def test_run_configuration_logs_actions_error():
     flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock())
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    expected_results = [flexmock()]
-    flexmock(module).should_receive('log_error_records').and_return(expected_results)
+    expected_results = [flexmock(), flexmock()]
+    flexmock(module).should_receive('log_error_records').and_return(
+        expected_results[:1]
+    ).and_return(expected_results[1:])
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_actions').and_raise(OSError)
     flexmock(module.command).should_receive('filter_hooks')
@@ -532,7 +534,7 @@ def test_run_configuration_logs_on_error_hook_error():
     expected_results = [flexmock(), flexmock()]
     flexmock(module).should_receive('log_error_records').and_return(
         expected_results[:1]
-    ).and_return(expected_results[1:])
+    ).and_return(expected_results[1:2]).and_return(expected_results[2:])
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_actions').and_raise(OSError)
     config = {'repositories': [{'path': 'foo'}]}
@@ -601,8 +603,10 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
     flexmock(module.command).should_receive('filter_hooks')
     flexmock(module.command).should_receive('execute_hooks').and_raise(error)
-    expected_results = [flexmock()]
-    flexmock(module).should_receive('log_error_records').and_return(expected_results)
+    expected_results = [flexmock(), flexmock()]
+    flexmock(module).should_receive('log_error_records').and_return(
+        expected_results[:1]
+    ).and_return(expected_results[1:])
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_actions').and_raise(OSError)
     config = {'repositories': [{'path': 'foo'}]}
@@ -654,11 +658,14 @@ def test_run_configuration_retries_hard_error():
         levelno=logging.WARNING,
         log_command_error_output=True,
     ).and_return([flexmock()])
-    error_logs = [flexmock()]
+    error_logs = [flexmock(), flexmock()]
     flexmock(module).should_receive('log_error_records').with_args(
         'Error running actions for repository',
         OSError,
-    ).and_return(error_logs)
+    ).and_return(error_logs[:1]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'Error running configuration',
+    ).and_return(error_logs[1:]).ordered()
     flexmock(module.command).should_receive('filter_hooks')
     flexmock(module.command).should_receive('execute_hooks')
     config = {'repositories': [{'path': 'foo'}], 'retries': 1}
@@ -680,13 +687,16 @@ def test_run_configuration_retries_repositories_in_order():
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
-    expected_results = [flexmock(), flexmock()]
+    expected_results = [flexmock(), flexmock(), flexmock()]
     flexmock(module).should_receive('log_error_records').with_args(
         'Error running actions for repository', OSError
     ).and_return(expected_results[:1]).ordered()
     flexmock(module).should_receive('log_error_records').with_args(
         'Error running actions for repository', OSError
-    ).and_return(expected_results[1:]).ordered()
+    ).and_return(expected_results[1:2]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'Error running configuration',
+    ).and_return(expected_results[2:]).ordered()
     flexmock(module.command).should_receive('filter_hooks')
     flexmock(module.command).should_receive('execute_hooks')
     config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}
@@ -728,6 +738,10 @@ def test_run_configuration_retries_round_robin():
     flexmock(module).should_receive('log_error_records').with_args(
         'Error running actions for repository', OSError
     ).and_return(bar_error_logs).ordered()
+    config_error_logs = [flexmock()]
+    flexmock(module).should_receive('log_error_records').with_args(
+        'Error running configuration',
+    ).and_return(config_error_logs).ordered()
     flexmock(module.command).should_receive('filter_hooks')
     flexmock(module.command).should_receive('execute_hooks')
     config = {
@@ -741,7 +755,7 @@ def test_run_configuration_retries_round_robin():
 
     results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments))
 
-    assert results == foo_error_logs + bar_error_logs
+    assert results == foo_error_logs + bar_error_logs + config_error_logs
 
 
 def test_run_configuration_with_one_retry():
@@ -766,10 +780,13 @@ def test_run_configuration_with_one_retry():
         levelno=logging.WARNING,
         log_command_error_output=True,
     ).and_return(flexmock()).ordered()
-    error_logs = [flexmock()]
+    error_logs = [flexmock(), flexmock()]
     flexmock(module).should_receive('log_error_records').with_args(
         'Error running actions for repository', OSError
-    ).and_return(error_logs).ordered()
+    ).and_return(error_logs[:1]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'Error running configuration',
+    ).and_return(error_logs[1:]).ordered()
     flexmock(module.command).should_receive('filter_hooks')
     flexmock(module.command).should_receive('execute_hooks')
     config = {
@@ -818,10 +835,13 @@ def test_run_configuration_with_retry_wait_does_backoff_after_each_retry():
     ).and_return([flexmock()]).ordered()
 
     flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
-    error_logs = [flexmock()]
+    error_logs = [flexmock(), flexmock()]
     flexmock(module).should_receive('log_error_records').with_args(
         'Error running actions for repository', OSError
-    ).and_return(error_logs).ordered()
+    ).and_return(error_logs[:1]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'Error running configuration',
+    ).and_return(error_logs[1:]).ordered()
     flexmock(module.command).should_receive('filter_hooks')
     flexmock(module.command).should_receive('execute_hooks')
     config = {
@@ -867,10 +887,13 @@ def test_run_configuration_with_multiple_repositories_retries_with_timeout():
 
     # Sleep before retrying bar (and failing)
     flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
-    error_logs = [flexmock()]
+    error_logs = [flexmock(), flexmock()]
     flexmock(module).should_receive('log_error_records').with_args(
         'Error running actions for repository', OSError
-    ).and_return(error_logs).ordered()
+    ).and_return(error_logs[:1]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'Error running configuration',
+    ).and_return(error_logs[1:]).ordered()
     flexmock(module.command).should_receive('filter_hooks')
     flexmock(module.command).should_receive('execute_hooks')
     config = {
@@ -891,6 +914,9 @@ def test_run_configuration_with_multiple_repositories_retries_with_timeout():
 def test_run_actions_runs_repo_create():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.repo_create).should_receive('run_repo_create').once()
 
@@ -914,6 +940,9 @@ def test_run_actions_runs_repo_create():
 def test_run_actions_adds_label_file_to_hook_context():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     expected = flexmock()
     flexmock(borgmatic.actions.create).should_receive('run_create').with_args(
@@ -947,6 +976,9 @@ def test_run_actions_adds_label_file_to_hook_context():
 def test_run_actions_adds_log_file_to_hook_context():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     expected = flexmock()
     flexmock(borgmatic.actions.create).should_receive('run_create').with_args(
@@ -980,6 +1012,9 @@ def test_run_actions_adds_log_file_to_hook_context():
 def test_run_actions_runs_transfer():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.transfer).should_receive('run_transfer').once()
 
@@ -1000,6 +1035,9 @@ def test_run_actions_runs_transfer():
 def test_run_actions_runs_create():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     expected = flexmock()
     flexmock(borgmatic.actions.create).should_receive('run_create').and_yield(expected).once()
@@ -1019,10 +1057,26 @@ def test_run_actions_runs_create():
     assert result == (expected,)
 
 
-def test_run_actions_with_skip_actions_skips_create():
+def test_run_actions_with_skip_actions_does_not_run_action_or_action_command_hooks():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return(['create'])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
+    flexmock(module.command).should_receive('Before_after_hooks').with_args(
+        command_hooks=object,
+        before_after='action',
+        umask=object,
+        working_directory=object,
+        dry_run=object,
+        action_names=object,
+        configuration_filename=object,
+        repository_label=object,
+        log_file=object,
+        repositories=object,
+        repository=object,
+    ).never()
     flexmock(borgmatic.actions.create).should_receive('run_create').never()
 
     tuple(
@@ -1042,6 +1096,9 @@ def test_run_actions_with_skip_actions_skips_create():
 def test_run_actions_runs_recreate():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
 
     flexmock(borgmatic.actions.recreate).should_receive('run_recreate').once()
@@ -1060,29 +1117,12 @@ def test_run_actions_runs_recreate():
     )
 
 
-def test_run_actions_with_skip_actions_skips_recreate():
-    flexmock(module).should_receive('add_custom_log_levels')
-    flexmock(module).should_receive('get_skip_actions').and_return(['recreate'])
-    flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
-    flexmock(borgmatic.actions.recreate).should_receive('run_recreate').never()
-
-    tuple(
-        module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'recreate': flexmock()},
-            config_filename=flexmock(),
-            config={'repositories': [], 'skip_actions': ['recreate']},
-            config_paths=[],
-            local_path=flexmock(),
-            remote_path=flexmock(),
-            local_borg_version=flexmock(),
-            repository={'path': 'repo'},
-        )
-    )
-
-
 def test_run_actions_runs_prune():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.prune).should_receive('run_prune').once()
 
@@ -1100,29 +1140,12 @@ def test_run_actions_runs_prune():
     )
 
 
-def test_run_actions_with_skip_actions_skips_prune():
-    flexmock(module).should_receive('add_custom_log_levels')
-    flexmock(module).should_receive('get_skip_actions').and_return(['prune'])
-    flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
-    flexmock(borgmatic.actions.prune).should_receive('run_prune').never()
-
-    tuple(
-        module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'prune': flexmock()},
-            config_filename=flexmock(),
-            config={'repositories': [], 'skip_actions': ['prune']},
-            config_paths=[],
-            local_path=flexmock(),
-            remote_path=flexmock(),
-            local_borg_version=flexmock(),
-            repository={'path': 'repo'},
-        )
-    )
-
-
 def test_run_actions_runs_compact():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.compact).should_receive('run_compact').once()
 
@@ -1140,29 +1163,12 @@ def test_run_actions_runs_compact():
     )
 
 
-def test_run_actions_with_skip_actions_skips_compact():
-    flexmock(module).should_receive('add_custom_log_levels')
-    flexmock(module).should_receive('get_skip_actions').and_return(['compact'])
-    flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
-    flexmock(borgmatic.actions.compact).should_receive('run_compact').never()
-
-    tuple(
-        module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'compact': flexmock()},
-            config_filename=flexmock(),
-            config={'repositories': [], 'skip_actions': ['compact']},
-            config_paths=[],
-            local_path=flexmock(),
-            remote_path=flexmock(),
-            local_borg_version=flexmock(),
-            repository={'path': 'repo'},
-        )
-    )
-
-
 def test_run_actions_runs_check_when_repository_enabled_for_checks():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True)
     flexmock(borgmatic.actions.check).should_receive('run_check').once()
@@ -1184,6 +1190,9 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks():
 def test_run_actions_skips_check_when_repository_not_enabled_for_checks():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(False)
     flexmock(borgmatic.actions.check).should_receive('run_check').never()
@@ -1202,30 +1211,12 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks():
     )
 
 
-def test_run_actions_with_skip_actions_skips_check():
-    flexmock(module).should_receive('add_custom_log_levels')
-    flexmock(module).should_receive('get_skip_actions').and_return(['check'])
-    flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
-    flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True)
-    flexmock(borgmatic.actions.check).should_receive('run_check').never()
-
-    tuple(
-        module.run_actions(
-            arguments={'global': flexmock(dry_run=False), 'check': flexmock()},
-            config_filename=flexmock(),
-            config={'repositories': [], 'skip_actions': ['check']},
-            config_paths=[],
-            local_path=flexmock(),
-            remote_path=flexmock(),
-            local_borg_version=flexmock(),
-            repository={'path': 'repo'},
-        )
-    )
-
-
 def test_run_actions_runs_extract():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.extract).should_receive('run_extract').once()
 
@@ -1246,6 +1237,9 @@ def test_run_actions_runs_extract():
 def test_run_actions_runs_export_tar():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.export_tar).should_receive('run_export_tar').once()
 
@@ -1266,6 +1260,9 @@ def test_run_actions_runs_export_tar():
 def test_run_actions_runs_mount():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.mount).should_receive('run_mount').once()
 
@@ -1286,6 +1283,9 @@ def test_run_actions_runs_mount():
 def test_run_actions_runs_restore():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.restore).should_receive('run_restore').once()
 
@@ -1306,6 +1306,9 @@ def test_run_actions_runs_restore():
 def test_run_actions_runs_repo_list():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     expected = flexmock()
     flexmock(borgmatic.actions.repo_list).should_receive('run_repo_list').and_yield(expected).once()
@@ -1328,6 +1331,9 @@ def test_run_actions_runs_repo_list():
 def test_run_actions_runs_list():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     expected = flexmock()
     flexmock(borgmatic.actions.list).should_receive('run_list').and_yield(expected).once()
@@ -1350,6 +1356,9 @@ def test_run_actions_runs_list():
 def test_run_actions_runs_repo_info():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     expected = flexmock()
     flexmock(borgmatic.actions.repo_info).should_receive('run_repo_info').and_yield(expected).once()
@@ -1372,6 +1381,9 @@ def test_run_actions_runs_repo_info():
 def test_run_actions_runs_info():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     expected = flexmock()
     flexmock(borgmatic.actions.info).should_receive('run_info').and_yield(expected).once()
@@ -1394,6 +1406,9 @@ def test_run_actions_runs_info():
 def test_run_actions_runs_break_lock():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.break_lock).should_receive('run_break_lock').once()
 
@@ -1414,6 +1429,9 @@ def test_run_actions_runs_break_lock():
 def test_run_actions_runs_export_key():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.export_key).should_receive('run_export_key').once()
 
@@ -1434,6 +1452,9 @@ def test_run_actions_runs_export_key():
 def test_run_actions_runs_import_key():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.import_key).should_receive('run_import_key').once()
 
@@ -1454,6 +1475,9 @@ def test_run_actions_runs_import_key():
 def test_run_actions_runs_change_passphrase():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.change_passphrase).should_receive('run_change_passphrase').once()
 
@@ -1477,6 +1501,9 @@ def test_run_actions_runs_change_passphrase():
 def test_run_actions_runs_delete():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.delete).should_receive('run_delete').once()
 
@@ -1497,6 +1524,9 @@ def test_run_actions_runs_delete():
 def test_run_actions_runs_repo_delete():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.repo_delete).should_receive('run_repo_delete').once()
 
@@ -1520,6 +1550,9 @@ def test_run_actions_runs_repo_delete():
 def test_run_actions_runs_borg():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.borg).should_receive('run_borg').once()
 
@@ -1540,6 +1573,9 @@ def test_run_actions_runs_borg():
 def test_run_actions_runs_multiple_actions_in_argument_order():
     flexmock(module).should_receive('add_custom_log_levels')
     flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
     flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
     flexmock(borgmatic.actions.borg).should_receive('run_borg').once().ordered()
     flexmock(borgmatic.actions.restore).should_receive('run_restore').once().ordered()
@@ -1562,6 +1598,50 @@ def test_run_actions_runs_multiple_actions_in_argument_order():
     )
 
 
+def test_run_actions_runs_action_hooks_for_one_action_at_a_time():
+    flexmock(module).should_receive('add_custom_log_levels')
+    flexmock(module).should_receive('get_skip_actions').and_return([])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock())
+
+    for action_name in ('borg', 'restore'):
+        flexmock(module.command).should_receive('Before_after_hooks').with_args(
+            command_hooks=object,
+            before_after='action',
+            umask=object,
+            working_directory=object,
+            dry_run=object,
+            action_names=(action_name,),
+            configuration_filename=object,
+            repository_label=object,
+            log_file=object,
+            repositories=object,
+            repository=object,
+        ).and_return(flexmock()).once()
+
+    flexmock(borgmatic.actions.borg).should_receive('run_borg')
+    flexmock(borgmatic.actions.restore).should_receive('run_restore')
+
+    tuple(
+        module.run_actions(
+            arguments={
+                'global': flexmock(dry_run=False),
+                'borg': flexmock(),
+                'restore': flexmock(),
+            },
+            config_filename=flexmock(),
+            config={'repositories': []},
+            config_paths=[],
+            local_path=flexmock(),
+            remote_path=flexmock(),
+            local_borg_version=flexmock(),
+            repository={'path': 'repo'},
+        )
+    )
+
+
 @pytest.mark.parametrize(
     'resolve_env',
     ((True, False),),
@@ -1852,7 +1932,12 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure():
 
 def test_collect_configuration_run_summary_logs_info_for_success():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['finish']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return([])
@@ -1871,9 +1956,17 @@ def test_collect_configuration_run_summary_logs_info_for_success():
 
 
 def test_collect_configuration_run_summary_executes_hooks_for_create():
+    before_everything_hook = {'before': 'everything', 'run': ['echo hi']}
+    after_everything_hook = {'after': 'everything', 'run': ['echo hi']}
+    command_hooks = (before_everything_hook, after_everything_hook)
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
-    flexmock(module.command).should_receive('execute_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        command_hooks, before='everything', action_names=object
+    ).and_return([before_everything_hook])
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        command_hooks, after='everything', action_names=object, state_names=['finish']
+    ).and_return([after_everything_hook])
+    flexmock(module.command).should_receive('execute_hooks').twice()
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return([])
     arguments = {
@@ -1883,7 +1976,7 @@ def test_collect_configuration_run_summary_executes_hooks_for_create():
 
     logs = tuple(
         module.collect_configuration_run_summary_logs(
-            {'test.yaml': {}},
+            {'test.yaml': {'commands': command_hooks}},
             config_paths=['/tmp/test.yaml'],
             arguments=arguments,
             log_file_path=None,
@@ -1893,9 +1986,51 @@ def test_collect_configuration_run_summary_executes_hooks_for_create():
     assert {log.levelno for log in logs} == {logging.INFO}
 
 
+def test_collect_configuration_run_summary_deduplicates_everything_hooks_across_config_files():
+    before_everything_hook = {'before': 'everything', 'run': ['echo hi']}
+    after_everything_hook = {'after': 'everything', 'run': ['echo hi']}
+    command_hooks = (before_everything_hook, after_everything_hook)
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        command_hooks, before='everything', action_names=object
+    ).and_return([before_everything_hook]).once()
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        (after_everything_hook,), before='everything', action_names=object
+    ).and_return([]).once()
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        command_hooks, after='everything', action_names=object, state_names=['finish']
+    ).and_return([after_everything_hook]).once()
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        (before_everything_hook,), after='everything', action_names=object, state_names=['finish']
+    ).and_return([]).once()
+    flexmock(module.command).should_receive('execute_hooks').twice()
+    flexmock(module).should_receive('Log_prefix').and_return(flexmock())
+    flexmock(module).should_receive('run_configuration').and_return([])
+    arguments = {
+        'create': flexmock(),
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs(
+            {'test.yaml': {'commands': command_hooks}, 'other.yaml': {'commands': command_hooks}},
+            config_paths=['/tmp/test.yaml', '/tmp/other.yaml'],
+            arguments=arguments,
+            log_file_path=None,
+        )
+    )
+
+    assert {log.levelno for log in logs} == {logging.INFO}
+
+
 def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['finish']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return([])
@@ -1938,7 +2073,12 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error():
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_mount():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['finish']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return([])
@@ -1984,7 +2124,12 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error():
 
 def test_collect_configuration_run_summary_logs_missing_configs_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['fail']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     arguments = {'global': flexmock(config_paths=[])}
     expected_logs = (flexmock(),)
@@ -1999,9 +2144,17 @@ def test_collect_configuration_run_summary_logs_missing_configs_error():
     assert logs == expected_logs
 
 
-def test_collect_configuration_run_summary_logs_pre_hook_error():
+def test_collect_configuration_run_summary_logs_before_hook_error():
+    before_everything_hook = {'before': 'everything', 'run': ['echo hi']}
+    after_everything_hook = {'after': 'everything', 'run': ['echo hi']}
+    command_hooks = (before_everything_hook, after_everything_hook)
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    ).and_return([before_everything_hook])
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['fail']
+    ).and_return([after_everything_hook])
     flexmock(module.command).should_receive('execute_hooks').and_raise(ValueError)
     expected_logs = (flexmock(),)
     flexmock(module).should_receive('log_error_records').and_return(expected_logs)
@@ -2012,7 +2165,7 @@ def test_collect_configuration_run_summary_logs_pre_hook_error():
 
     logs = tuple(
         module.collect_configuration_run_summary_logs(
-            {'test.yaml': {}},
+            {'test.yaml': {'commands': command_hooks}},
             config_paths=['/tmp/test.yaml'],
             arguments=arguments,
             log_file_path=None,
@@ -2022,9 +2175,17 @@ def test_collect_configuration_run_summary_logs_pre_hook_error():
     assert logs == expected_logs
 
 
-def test_collect_configuration_run_summary_logs_post_hook_error():
+def test_collect_configuration_run_summary_logs_after_hook_error():
+    before_everything_hook = {'before': 'everything', 'run': ['echo hi']}
+    after_everything_hook = {'after': 'everything', 'run': ['echo hi']}
+    command_hooks = (before_everything_hook, after_everything_hook)
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    ).and_return([before_everything_hook])
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['finish']
+    ).and_return([after_everything_hook])
     flexmock(module.command).should_receive('execute_hooks').and_return(None).and_raise(ValueError)
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return([])
@@ -2037,7 +2198,7 @@ def test_collect_configuration_run_summary_logs_post_hook_error():
 
     logs = tuple(
         module.collect_configuration_run_summary_logs(
-            {'test.yaml': {}},
+            {'test.yaml': {'commands': command_hooks}},
             config_paths=['/tmp/test.yaml'],
             arguments=arguments,
             log_file_path=None,
@@ -2072,7 +2233,12 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_list():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['finish']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return([])
@@ -2095,7 +2261,12 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list():
 
 def test_collect_configuration_run_summary_logs_run_configuration_error_logs():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['fail']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return(
@@ -2118,7 +2289,12 @@ def test_collect_configuration_run_summary_logs_run_configuration_error_logs():
 
 def test_collect_configuration_run_summary_logs_run_umount_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['fail']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return([])
@@ -2145,7 +2321,12 @@ def test_collect_configuration_run_summary_logs_run_umount_error():
 
 def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module.command).should_receive('filter_hooks')
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, before='everything', action_names=object
+    )
+    flexmock(module.command).should_receive('filter_hooks').with_args(
+        object, after='everything', action_names=object, state_names=['finish']
+    )
     flexmock(module.command).should_receive('execute_hooks')
     flexmock(module).should_receive('Log_prefix').and_return(flexmock())
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(

tests/unit/hooks/test_command.py

@@ -11,8 +11,16 @@ def test_interpolate_context_passes_through_command_without_variable():
     assert module.interpolate_context('pre-backup', 'ls', {'foo': 'bar'}) == 'ls'
 
 
-def test_interpolate_context_passes_through_command_with_unknown_variable():
+def test_interpolate_context_warns_and_passes_through_command_with_unknown_variable():
     command = 'ls {baz}'  # noqa: FS003
+    flexmock(module.logger).should_receive('warning').once()
+
+    assert module.interpolate_context('pre-backup', command, {'foo': 'bar'}) == command
+
+
+def test_interpolate_context_does_not_warn_and_passes_through_command_with_unknown_variable_matching_borg_placeholder():
+    command = 'ls {hostname}'  # noqa: FS003
+    flexmock(module.logger).should_receive('warning').never()
 
     assert module.interpolate_context('pre-backup', command, {'foo': 'bar'}) == command
 
@@ -167,6 +175,44 @@ def test_make_environment_with_pyinstaller_and_LD_LIBRARY_PATH_ORIG_copies_it_in
                 },
             ),
         ),
+        (
+            (
+                {
+                    'before': 'action',
+                    'states': ['finish'],  # Not actually valid; only valid for "after".
+                    'run': ['foo'],
+                },
+                {
+                    'after': 'action',
+                    'run': ['bar'],
+                },
+                {
+                    'after': 'action',
+                    'states': ['finish'],
+                    'run': ['baz'],
+                },
+                {
+                    'after': 'action',
+                    'states': ['fail'],
+                    'run': ['quux'],
+                },
+            ),
+            {
+                'after': 'action',
+                'state_names': ['finish'],
+            },
+            (
+                {
+                    'after': 'action',
+                    'run': ['bar'],
+                },
+                {
+                    'after': 'action',
+                    'states': ['finish'],
+                    'run': ['baz'],
+                },
+            ),
+        ),
     ),
 )
 def test_filter_hooks(hooks, filters, expected_hooks):
@@ -336,14 +382,13 @@ def test_before_after_hooks_calls_command_hooks():
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         before='action',
-        hook_name='myhook',
         action_names=['create'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         after='action',
-        hook_name='myhook',
         action_names=['create'],
+        state_names=['finish'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('execute_hooks').twice()
 
@@ -353,7 +398,6 @@ def test_before_after_hooks_calls_command_hooks():
         umask=1234,
         working_directory='/working',
         dry_run=False,
-        hook_name='myhook',
         action_names=['create'],
         context1='stuff',
         context2='such',
@@ -369,14 +413,13 @@ def test_before_after_hooks_with_before_error_runs_after_hook_and_raises():
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         before='action',
-        hook_name='myhook',
         action_names=['create'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         after='action',
-        hook_name='myhook',
         action_names=['create'],
+        state_names=['fail'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('execute_hooks').and_raise(OSError).and_return(None)
     flexmock(module).should_receive('considered_soft_failure').and_return(False)
@@ -388,7 +431,6 @@ def test_before_after_hooks_with_before_error_runs_after_hook_and_raises():
             umask=1234,
             working_directory='/working',
             dry_run=False,
-            hook_name='myhook',
             action_names=['create'],
             context1='stuff',
             context2='such',
@@ -396,7 +438,7 @@ def test_before_after_hooks_with_before_error_runs_after_hook_and_raises():
             assert False  # This should never get called.
 
 
-def test_before_after_hooks_with_before_soft_failure_does_not_raise():
+def test_before_after_hooks_with_before_soft_failure_raises():
     commands = [
         {'before': 'repository', 'run': ['foo', 'bar']},
         {'after': 'repository', 'run': ['baz']},
@@ -404,30 +446,61 @@ def test_before_after_hooks_with_before_soft_failure_does_not_raise():
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         before='action',
-        hook_name='myhook',
         action_names=['create'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         after='action',
-        hook_name='myhook',
         action_names=['create'],
-    ).and_return(flexmock()).once()
+        state_names=['finish'],
+    ).never()
     flexmock(module).should_receive('execute_hooks').and_raise(OSError)
     flexmock(module).should_receive('considered_soft_failure').and_return(True)
 
-    with module.Before_after_hooks(
-        command_hooks=commands,
-        before_after='action',
-        umask=1234,
-        working_directory='/working',
-        dry_run=False,
-        hook_name='myhook',
+    with pytest.raises(OSError):
+        with module.Before_after_hooks(
+            command_hooks=commands,
+            before_after='action',
+            umask=1234,
+            working_directory='/working',
+            dry_run=False,
+            action_names=['create'],
+            context1='stuff',
+            context2='such',
+        ):
+            pass
+
+
+def test_before_after_hooks_with_wrapped_code_error_runs_after_hook_and_raises():
+    commands = [
+        {'before': 'repository', 'run': ['foo', 'bar']},
+        {'after': 'repository', 'run': ['baz']},
+    ]
+    flexmock(module).should_receive('filter_hooks').with_args(
+        commands,
+        before='action',
         action_names=['create'],
-        context1='stuff',
-        context2='such',
-    ):
-        pass
+    ).and_return(flexmock()).once()
+    flexmock(module).should_receive('filter_hooks').with_args(
+        commands,
+        after='action',
+        action_names=['create'],
+        state_names=['fail'],
+    ).and_return(flexmock()).once()
+    flexmock(module).should_receive('execute_hooks').twice()
+
+    with pytest.raises(ValueError):
+        with module.Before_after_hooks(
+            command_hooks=commands,
+            before_after='action',
+            umask=1234,
+            working_directory='/working',
+            dry_run=False,
+            action_names=['create'],
+            context1='stuff',
+            context2='such',
+        ):
+            raise ValueError()
 
 
 def test_before_after_hooks_with_after_error_raises():
@@ -438,14 +511,13 @@ def test_before_after_hooks_with_after_error_raises():
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         before='action',
-        hook_name='myhook',
         action_names=['create'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         after='action',
-        hook_name='myhook',
         action_names=['create'],
+        state_names=['finish'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('execute_hooks').and_return(None).and_raise(OSError)
     flexmock(module).should_receive('considered_soft_failure').and_return(False)
@@ -457,7 +529,6 @@ def test_before_after_hooks_with_after_error_raises():
             umask=1234,
             working_directory='/working',
             dry_run=False,
-            hook_name='myhook',
             action_names=['create'],
             context1='stuff',
             context2='such',
@@ -465,7 +536,7 @@ def test_before_after_hooks_with_after_error_raises():
             pass
 
 
-def test_before_after_hooks_with_after_soft_failure_does_not_raise():
+def test_before_after_hooks_with_after_soft_failure_raises():
     commands = [
         {'before': 'repository', 'run': ['foo', 'bar']},
         {'after': 'repository', 'run': ['baz']},
@@ -473,30 +544,29 @@ def test_before_after_hooks_with_after_soft_failure_does_not_raise():
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         before='action',
-        hook_name='myhook',
         action_names=['create'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('filter_hooks').with_args(
         commands,
         after='action',
-        hook_name='myhook',
         action_names=['create'],
+        state_names=['finish'],
     ).and_return(flexmock()).once()
     flexmock(module).should_receive('execute_hooks').and_return(None).and_raise(OSError)
     flexmock(module).should_receive('considered_soft_failure').and_return(True)
 
-    with module.Before_after_hooks(
-        command_hooks=commands,
-        before_after='action',
-        umask=1234,
-        working_directory='/working',
-        dry_run=False,
-        hook_name='myhook',
-        action_names=['create'],
-        context1='stuff',
-        context2='such',
-    ):
-        pass
+    with pytest.raises(OSError):
+        with module.Before_after_hooks(
+            command_hooks=commands,
+            before_after='action',
+            umask=1234,
+            working_directory='/working',
+            dry_run=False,
+            action_names=['create'],
+            context1='stuff',
+            context2='such',
+        ):
+            pass
 
 
 def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail():
@@ -513,3 +583,11 @@ def test_considered_soft_failure_does_not_treat_other_exit_code_as_soft_fail():
 
 def test_considered_soft_failure_does_not_treat_other_exception_type_as_soft_fail():
     assert not module.considered_soft_failure(Exception())
+
+
+def test_considered_soft_failure_caches_results_and_only_logs_once():
+    error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again')
+    flexmock(module.logger).should_receive('info').once()
+
+    assert module.considered_soft_failure(error)
+    assert module.considered_soft_failure(error)