Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/469

NEWS

@@ -1,3 +1,35 @@
+1.6.0
+ * #381: BREAKING: Greatly simplify configuration file reuse by deep merging when including common
+   configuration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#include-merging
+ * #473: BREAKING: Instead of executing "before" command hooks before all borgmatic actions run (and
+   "after" hooks after), execute these hooks right before/after the corresponding action. E.g.,
+   "before_check" now runs immediately before the "check" action. This better supports running
+   timing-sensitive tasks like pausing containers. Side effect: before/after command hooks now run
+   once for each configured repository instead of once per configuration file. Additionally, the
+   "repositories" interpolated variable has been changed to "repository", containing the path to the
+   current repository for the hook.
+ * #513: Add mention of sudo's "secure_path" option to borgmatic installation documentation.
+ * #515: Fix "borgmatic borg key ..." to pass parameters to Borg in the correct order.
+ * #516: Fix handling of TERM signal to exit borgmatic, not just forward the signal to Borg.
+ * #517: Fix borgmatic exit code (so it's zero) when initial Borg calls fail but later retries
+   succeed.
+ * Change Healthchecks logs truncation size from 10k bytes to 100k bytes, corresponding to that
+   same change on Healthchecks.io.
+
+1.5.24
+ * #431: Add "working_directory" option to support source directories with relative paths.
+ * #444: When loading a configuration file that is unreadable due to file permissions, warn instead
+   of erroring. This supports running borgmatic as a non-root user with configuration in ~/.config
+   even if there is an unreadable global configuration file in /etc.
+ * #469: Add "repositories" context to "before_*" and "after_*" command action hooks. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+ * #486: Fix handling of "patterns_from" and "exclude_from" options to error instead of warning when
+   referencing unreadable files and "create" action is run.
+ * #507: Fix Borg usage error in the "compact" action when running "borgmatic --dry-run". Now, skip
+   "compact" entirely during a dry run.
+
 1.5.23
  * #394: Compact repository segments and free space with new "borgmatic compact" action. Borg 1.2+
    only. Also run "compact" by default when no actions are specified, as "prune" in Borg 1.2 no

borgmatic/borg/borg.py

@@ -21,17 +21,20 @@ def run_arbitrary_borg(
 
     try:
         options = options[1:] if options[0] == '--' else options
-        borg_command = options[0]
-        command_options = tuple(options[1:])
+
+        # Borg's "key" command has a sub-command ("export", etc.) that must follow it.
+        command_options_start_index = 2 if options[0] == 'key' else 1
+        borg_command = tuple(options[:command_options_start_index])
+        command_options = tuple(options[command_options_start_index:])
     except IndexError:
-        borg_command = None
+        borg_command = ()
         command_options = ()
 
     repository_archive = '::'.join((repository, archive)) if repository and archive else repository
 
     full_command = (
         (local_path,)
-        + ((borg_command,) if borg_command else ())
+        + borg_command
         + ((repository_archive,) if borg_command and repository_archive else ())
         + command_options
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())

borgmatic/borg/compact.py

@@ -33,9 +33,9 @@ def compact_segments(
         + (('--threshold', str(threshold)) if threshold else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--dry-run',) if dry_run else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + (repository,)
     )
 
-    execute_command(full_command, output_log_level=logging.INFO, borg_local_path=local_path)
+    if not dry_run:
+        execute_command(full_command, output_log_level=logging.INFO, borg_local_path=local_path)

borgmatic/borg/create.py

@@ -11,7 +11,7 @@ from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_w
 logger = logging.getLogger(__name__)
 
 
-def _expand_directory(directory):
+def expand_directory(directory):
     '''
     Given a directory path, expand any tilde (representing a user's home directory) and any globs
     therein. Return a list of one or more resulting paths.
@@ -21,7 +21,7 @@ def _expand_directory(directory):
     return glob.glob(expanded_directory) or [expanded_directory]
 
 
-def _expand_directories(directories):
+def expand_directories(directories):
     '''
     Given a sequence of directory paths, expand tildes and globs in each one. Return all the
     resulting directories as a single flattened tuple.
@@ -30,11 +30,11 @@ def _expand_directories(directories):
         return ()
 
     return tuple(
-        itertools.chain.from_iterable(_expand_directory(directory) for directory in directories)
+        itertools.chain.from_iterable(expand_directory(directory) for directory in directories)
     )
 
 
-def _expand_home_directories(directories):
+def expand_home_directories(directories):
     '''
     Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
     Return the results as a tuple.
@@ -98,7 +98,7 @@ def deduplicate_directories(directory_devices):
     return tuple(sorted(deduplicated))
 
 
-def _write_pattern_file(patterns=None):
+def write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
     if no patterns are provided.
@@ -113,7 +113,19 @@ def _write_pattern_file(patterns=None):
     return pattern_file
 
 
-def _make_pattern_flags(location_config, pattern_filename=None):
+def ensure_files_readable(*filename_lists):
+    '''
+    Given a sequence of filename sequences, ensure that each filename is openable. This prevents
+    unreadable files from being passed to Borg, which in certain situations only warns instead of
+    erroring.
+    '''
+    for file_object in itertools.chain.from_iterable(
+        filename_list for filename_list in filename_lists if filename_list
+    ):
+        open(file_object).close()
+
+
+def make_pattern_flags(location_config, pattern_filename=None):
     '''
     Given a location config dict with a potential patterns_from option, and a filename containing
     any additional patterns, return the corresponding Borg flags for those files as a tuple.
@@ -129,7 +141,7 @@ def _make_pattern_flags(location_config, pattern_filename=None):
     )
 
 
-def _make_exclude_flags(location_config, exclude_filename=None):
+def make_exclude_flags(location_config, exclude_filename=None):
     '''
     Given a location config dict with various exclude options, and a filename containing any exclude
     patterns, return the corresponding Borg flags as a tuple.
@@ -206,16 +218,20 @@ def create_archive(
     '''
     sources = deduplicate_directories(
         map_directories_to_devices(
-            _expand_directories(
+            expand_directories(
                 location_config['source_directories']
                 + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
             )
         )
     )
 
-    pattern_file = _write_pattern_file(location_config.get('patterns'))
-    exclude_file = _write_pattern_file(
-        _expand_home_directories(location_config.get('exclude_patterns'))
+    try:
+        working_directory = os.path.expanduser(location_config.get('working_directory'))
+    except TypeError:
+        working_directory = None
+    pattern_file = write_pattern_file(location_config.get('patterns'))
+    exclude_file = write_pattern_file(
+        expand_home_directories(location_config.get('exclude_patterns'))
     )
     checkpoint_interval = storage_config.get('checkpoint_interval', None)
     chunker_params = storage_config.get('chunker_params', None)
@@ -251,11 +267,13 @@ def create_archive(
             ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
         )
 
+    ensure_files_readable(location_config.get('patterns_from'), location_config.get('exclude_from'))
+
     full_command = (
         tuple(local_path.split(' '))
         + ('create',)
-        + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
-        + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
+        + make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
+        + make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
@@ -309,6 +327,13 @@ def create_archive(
             output_log_level,
             output_file,
             borg_local_path=local_path,
+            working_directory=working_directory,
         )
 
-    return execute_command(full_command, output_log_level, output_file, borg_local_path=local_path)
+    return execute_command(
+        full_command,
+        output_log_level,
+        output_file,
+        borg_local_path=local_path,
+        working_directory=working_directory,
+    )

borgmatic/commands/borgmatic.py

@@ -68,7 +68,7 @@ def run_configuration(config_filename, config, arguments):
     try:
         local_borg_version = borg_version.local_borg_version(local_path)
     except (OSError, CalledProcessError, ValueError) as error:
-        yield from make_error_log_records(
+        yield from log_error_records(
             '{}: Error getting local Borg version'.format(config_filename), error
         )
         return
@@ -83,46 +83,6 @@ def run_configuration(config_filename, config, arguments):
                 monitoring_log_level,
                 global_arguments.dry_run,
             )
-        if 'prune' in arguments:
-            command.execute_hook(
-                hooks.get('before_prune'),
-                hooks.get('umask'),
-                config_filename,
-                'pre-prune',
-                global_arguments.dry_run,
-            )
-        if 'compact' in arguments:
-            command.execute_hook(
-                hooks.get('before_compact'),
-                hooks.get('umask'),
-                config_filename,
-                'pre-compact',
-                global_arguments.dry_run,
-            )
-        if 'create' in arguments:
-            command.execute_hook(
-                hooks.get('before_backup'),
-                hooks.get('umask'),
-                config_filename,
-                'pre-backup',
-                global_arguments.dry_run,
-            )
-        if 'check' in arguments:
-            command.execute_hook(
-                hooks.get('before_check'),
-                hooks.get('umask'),
-                config_filename,
-                'pre-check',
-                global_arguments.dry_run,
-            )
-        if 'extract' in arguments:
-            command.execute_hook(
-                hooks.get('before_extract'),
-                hooks.get('umask'),
-                config_filename,
-                'pre-extract',
-                global_arguments.dry_run,
-            )
         if using_primary_action:
             dispatch.call_hooks(
                 'ping_monitor',
@@ -138,9 +98,7 @@ def run_configuration(config_filename, config, arguments):
             return
 
         encountered_error = error
-        yield from make_error_log_records(
-            '{}: Error running pre hook'.format(config_filename), error
-        )
+        yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error)
 
     if not encountered_error:
         repo_queue = Queue()
@@ -156,6 +114,7 @@ def run_configuration(config_filename, config, arguments):
             try:
                 yield from run_actions(
                     arguments=arguments,
+                    config_filename=config_filename,
                     location=location,
                     storage=storage,
                     retention=retention,
@@ -167,68 +126,32 @@ def run_configuration(config_filename, config, arguments):
                     repository_path=repository_path,
                 )
             except (OSError, CalledProcessError, ValueError) as error:
-                yield from make_error_log_records(
-                    '{}: Error running actions for repository'.format(repository_path), error
-                )
                 if retry_num < retries:
                     repo_queue.put((repository_path, retry_num + 1),)
+                    tuple(  # Consume the generator so as to trigger logging.
+                        log_error_records(
+                            '{}: Error running actions for repository'.format(repository_path),
+                            error,
+                            levelno=logging.WARNING,
+                            log_command_error_output=True,
+                        )
+                    )
                     logger.warning(
                         f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}'
                     )
                     continue
+
+                if command.considered_soft_failure(config_filename, error):
+                    return
+
+                yield from log_error_records(
+                    '{}: Error running actions for repository'.format(repository_path), error
+                )
                 encountered_error = error
                 error_repository = repository_path
 
     if not encountered_error:
         try:
-            if 'prune' in arguments:
-                command.execute_hook(
-                    hooks.get('after_prune'),
-                    hooks.get('umask'),
-                    config_filename,
-                    'post-prune',
-                    global_arguments.dry_run,
-                )
-            if 'compact' in arguments:
-                command.execute_hook(
-                    hooks.get('after_compact'),
-                    hooks.get('umask'),
-                    config_filename,
-                    'post-compact',
-                    global_arguments.dry_run,
-                )
-            if 'create' in arguments:
-                dispatch.call_hooks(
-                    'remove_database_dumps',
-                    hooks,
-                    config_filename,
-                    dump.DATABASE_HOOK_NAMES,
-                    location,
-                    global_arguments.dry_run,
-                )
-                command.execute_hook(
-                    hooks.get('after_backup'),
-                    hooks.get('umask'),
-                    config_filename,
-                    'post-backup',
-                    global_arguments.dry_run,
-                )
-            if 'check' in arguments:
-                command.execute_hook(
-                    hooks.get('after_check'),
-                    hooks.get('umask'),
-                    config_filename,
-                    'post-check',
-                    global_arguments.dry_run,
-                )
-            if 'extract' in arguments:
-                command.execute_hook(
-                    hooks.get('after_extract'),
-                    hooks.get('umask'),
-                    config_filename,
-                    'post-extract',
-                    global_arguments.dry_run,
-                )
             if using_primary_action:
                 dispatch.call_hooks(
                     'ping_monitor',
@@ -252,9 +175,7 @@ def run_configuration(config_filename, config, arguments):
                 return
 
             encountered_error = error
-            yield from make_error_log_records(
-                '{}: Error running post hook'.format(config_filename), error
-            )
+            yield from log_error_records('{}: Error pinging monitor'.format(config_filename), error)
 
     if encountered_error and using_primary_action:
         try:
@@ -289,7 +210,7 @@ def run_configuration(config_filename, config, arguments):
             if command.considered_soft_failure(config_filename, error):
                 return
 
-            yield from make_error_log_records(
+            yield from log_error_records(
                 '{}: Error running on-error hook'.format(config_filename), error
             )
 
@@ -297,6 +218,7 @@ def run_configuration(config_filename, config, arguments):
 def run_actions(
     *,
     arguments,
+    config_filename,
     location,
     storage,
     retention,
@@ -306,20 +228,28 @@ def run_actions(
     remote_path,
     local_borg_version,
     repository_path,
-):  # pragma: no cover
+):
     '''
-    Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
-    configuration dicts, local and remote paths to Borg, a local Borg version string, and a
-    repository name, run all actions from the command-line arguments on the given repository.
+    Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
+    filename, several different configuration dicts, local and remote paths to Borg, a local Borg
+    version string, and a repository name, run all actions from the command-line arguments on the
+    given repository.
 
     Yield JSON output strings from executing any actions that produce JSON.
 
     Raise OSError or subprocess.CalledProcessError if an error occurs running a command for an
-    action. Raise ValueError if the arguments or configuration passed to action are invalid.
+    action or a hook. Raise ValueError if the arguments or configuration passed to action are
+    invalid.
     '''
     repository = os.path.expanduser(repository_path)
     global_arguments = arguments['global']
     dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
+    hook_context = {
+        'repository': repository_path,
+        # Deprecated: For backwards compatibility with borgmatic < 1.6.0.
+        'repositories': ','.join(location['repositories']),
+    }
+
     if 'init' in arguments:
         logger.info('{}: Initializing repository'.format(repository))
         borg_init.initialize_repository(
@@ -332,6 +262,14 @@ def run_actions(
             remote_path=remote_path,
         )
     if 'prune' in arguments:
+        command.execute_hook(
+            hooks.get('before_prune'),
+            hooks.get('umask'),
+            config_filename,
+            'pre-prune',
+            global_arguments.dry_run,
+            **hook_context,
+        )
         logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
         borg_prune.prune_archives(
             global_arguments.dry_run,
@@ -343,7 +281,22 @@ def run_actions(
             stats=arguments['prune'].stats,
             files=arguments['prune'].files,
         )
+        command.execute_hook(
+            hooks.get('after_prune'),
+            hooks.get('umask'),
+            config_filename,
+            'post-prune',
+            global_arguments.dry_run,
+            **hook_context,
+        )
     if 'compact' in arguments:
+        command.execute_hook(
+            hooks.get('before_compact'),
+            hooks.get('umask'),
+            config_filename,
+            'pre-compact',
+            global_arguments.dry_run,
+        )
         if borg_feature.available(borg_feature.Feature.COMPACT, local_borg_version):
             logger.info('{}: Compacting segments{}'.format(repository, dry_run_label))
             borg_compact.compact_segments(
@@ -356,11 +309,26 @@ def run_actions(
                 cleanup_commits=arguments['compact'].cleanup_commits,
                 threshold=arguments['compact'].threshold,
             )
-        else:
+        else:  # pragma: nocover
             logger.info(
                 '{}: Skipping compact (only available/needed in Borg 1.2+)'.format(repository)
             )
+        command.execute_hook(
+            hooks.get('after_compact'),
+            hooks.get('umask'),
+            config_filename,
+            'post-compact',
+            global_arguments.dry_run,
+        )
     if 'create' in arguments:
+        command.execute_hook(
+            hooks.get('before_backup'),
+            hooks.get('umask'),
+            config_filename,
+            'pre-backup',
+            global_arguments.dry_run,
+            **hook_context,
+        )
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
         dispatch.call_hooks(
             'remove_database_dumps',
@@ -394,10 +362,35 @@ def run_actions(
             files=arguments['create'].files,
             stream_processes=stream_processes,
         )
-        if json_output:
+        if json_output:  # pragma: nocover
             yield json.loads(json_output)
 
+        dispatch.call_hooks(
+            'remove_database_dumps',
+            hooks,
+            config_filename,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
+        command.execute_hook(
+            hooks.get('after_backup'),
+            hooks.get('umask'),
+            config_filename,
+            'post-backup',
+            global_arguments.dry_run,
+            **hook_context,
+        )
+
     if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
+        command.execute_hook(
+            hooks.get('before_check'),
+            hooks.get('umask'),
+            config_filename,
+            'pre-check',
+            global_arguments.dry_run,
+            **hook_context,
+        )
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
             repository,
@@ -409,7 +402,23 @@ def run_actions(
             repair=arguments['check'].repair,
             only_checks=arguments['check'].only,
         )
+        command.execute_hook(
+            hooks.get('after_check'),
+            hooks.get('umask'),
+            config_filename,
+            'post-check',
+            global_arguments.dry_run,
+            **hook_context,
+        )
     if 'extract' in arguments:
+        command.execute_hook(
+            hooks.get('before_extract'),
+            hooks.get('umask'),
+            config_filename,
+            'pre-extract',
+            global_arguments.dry_run,
+            **hook_context,
+        )
         if arguments['extract'].repository is None or validate.repositories_match(
             repository, arguments['extract'].repository
         ):
@@ -432,6 +441,14 @@ def run_actions(
                 strip_components=arguments['extract'].strip_components,
                 progress=arguments['extract'].progress,
             )
+        command.execute_hook(
+            hooks.get('after_extract'),
+            hooks.get('umask'),
+            config_filename,
+            'post-extract',
+            global_arguments.dry_run,
+            **hook_context,
+        )
     if 'export-tar' in arguments:
         if arguments['export-tar'].repository is None or validate.repositories_match(
             repository, arguments['export-tar'].repository
@@ -464,7 +481,7 @@ def run_actions(
                 logger.info(
                     '{}: Mounting archive {}'.format(repository, arguments['mount'].archive)
                 )
-            else:
+            else:  # pragma: nocover
                 logger.info('{}: Mounting repository'.format(repository))
 
             borg_mount.mount_archive(
@@ -480,7 +497,7 @@ def run_actions(
                 local_path=local_path,
                 remote_path=remote_path,
             )
-    if 'restore' in arguments:
+    if 'restore' in arguments:  # pragma: nocover
         if arguments['restore'].repository is None or validate.repositories_match(
             repository, arguments['restore'].repository
         ):
@@ -579,7 +596,7 @@ def run_actions(
             repository, arguments['list'].repository
         ):
             list_arguments = copy.copy(arguments['list'])
-            if not list_arguments.json:
+            if not list_arguments.json:  # pragma: nocover
                 logger.warning('{}: Listing archives'.format(repository))
             list_arguments.archive = borg_list.resolve_archive_name(
                 repository, list_arguments.archive, storage, local_path, remote_path
@@ -591,14 +608,14 @@ def run_actions(
                 local_path=local_path,
                 remote_path=remote_path,
             )
-            if json_output:
+            if json_output:  # pragma: nocover
                 yield json.loads(json_output)
     if 'info' in arguments:
         if arguments['info'].repository is None or validate.repositories_match(
             repository, arguments['info'].repository
         ):
             info_arguments = copy.copy(arguments['info'])
-            if not info_arguments.json:
+            if not info_arguments.json:  # pragma: nocover
                 logger.warning('{}: Displaying summary info for archives'.format(repository))
             info_arguments.archive = borg_list.resolve_archive_name(
                 repository, info_arguments.archive, storage, local_path, remote_path
@@ -610,7 +627,7 @@ def run_actions(
                 local_path=local_path,
                 remote_path=remote_path,
             )
-            if json_output:
+            if json_output:  # pragma: nocover
                 yield json.loads(json_output)
     if 'borg' in arguments:
         if arguments['borg'].repository is None or validate.repositories_match(
@@ -646,6 +663,20 @@ def load_configurations(config_filenames, overrides=None):
             configs[config_filename] = validate.parse_configuration(
                 config_filename, validate.schema_filename(), overrides
             )
+        except PermissionError:
+            logs.extend(
+                [
+                    logging.makeLogRecord(
+                        dict(
+                            levelno=logging.WARNING,
+                            levelname='WARNING',
+                            msg='{}: Insufficient permissions to read configuration file'.format(
+                                config_filename
+                            ),
+                        )
+                    ),
+                ]
+            )
         except (ValueError, OSError, validate.Validation_error) as error:
             logs.extend(
                 [
@@ -678,28 +709,39 @@ def log_record(suppress_log=False, **kwargs):
     return record
 
 
-def make_error_log_records(message, error=None):
+def log_error_records(
+    message, error=None, levelno=logging.CRITICAL, log_command_error_output=False
+):
     '''
-    Given error message text and an optional exception object, yield a series of logging.LogRecord
-    instances with error summary information. As a side effect, log each record.
+    Given error message text, an optional exception object, an optional log level, and whether to
+    log the error output of a CalledProcessError (if any), log error summary information and also
+    yield it as a series of logging.LogRecord instances.
+
+    Note that because the logs are yielded as a generator, logs won't get logged unless you consume
+    the generator output.
     '''
+    level_name = logging._levelToName[levelno]
+
     if not error:
-        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        yield log_record(levelno=levelno, levelname=level_name, msg=message)
         return
 
     try:
         raise error
     except CalledProcessError as error:
-        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        yield log_record(levelno=levelno, levelname=level_name, msg=message)
         if error.output:
             # Suppress these logs for now and save full error output for the log summary at the end.
             yield log_record(
-                levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output, suppress_log=True
+                levelno=levelno,
+                levelname=level_name,
+                msg=error.output,
+                suppress_log=not log_command_error_output,
             )
-        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+        yield log_record(levelno=levelno, levelname=level_name, msg=error)
     except (ValueError, OSError) as error:
-        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
-        yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+        yield log_record(levelno=levelno, levelname=level_name, msg=message)
+        yield log_record(levelno=levelno, levelname=level_name, msg=error)
     except:  # noqa: E722
         # Raising above only as a means of determining the error type. Swallow the exception here
         # because we don't want the exception to propagate out of this function.
@@ -738,11 +780,11 @@ def collect_configuration_run_summary_logs(configs, arguments):
         try:
             validate.guard_configuration_contains_repository(repository, configs)
         except ValueError as error:
-            yield from make_error_log_records(str(error))
+            yield from log_error_records(str(error))
             return
 
     if not configs:
-        yield from make_error_log_records(
+        yield from log_error_records(
             '{}: No valid configuration files found'.format(
                 ' '.join(arguments['global'].config_paths)
             )
@@ -761,7 +803,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
                     arguments['global'].dry_run,
                 )
         except (CalledProcessError, ValueError, OSError) as error:
-            yield from make_error_log_records('Error running pre-everything hook', error)
+            yield from log_error_records('Error running pre-everything hook', error)
             return
 
     # Execute the actions corresponding to each configuration file.
@@ -771,7 +813,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
         error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
 
         if error_logs:
-            yield from make_error_log_records(
+            yield from log_error_records(
                 '{}: Error running configuration file'.format(config_filename)
             )
             yield from error_logs
@@ -793,7 +835,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
                 mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs)
             )
         except (CalledProcessError, OSError) as error:
-            yield from make_error_log_records('Error unmounting mount point', error)
+            yield from log_error_records('Error unmounting mount point', error)
 
     if json_results:
         sys.stdout.write(json.dumps(json_results))
@@ -810,7 +852,7 @@ def collect_configuration_run_summary_logs(configs, arguments):
                     arguments['global'].dry_run,
                 )
         except (CalledProcessError, ValueError, OSError) as error:
-            yield from make_error_log_records('Error running post-everything hook', error)
+            yield from log_error_records('Error running post-everything hook', error)
 
 
 def exit_with_help_link():  # pragma: no cover

borgmatic/config/load.py

@@ -28,6 +28,107 @@ def include_configuration(loader, filename_node):
     return load_configuration(os.path.expanduser(filename_node.value))
 
 
+DELETED_NODE = object()
+
+
+def deep_merge_nodes(nodes):
+    '''
+    Given a nested borgmatic configuration data structure as a list of tuples in the form of:
+
+        (
+            ruamel.yaml.nodes.ScalarNode as a key,
+            ruamel.yaml.nodes.MappingNode or other Node as a value,
+        ),
+
+    ... deep merge any node values corresponding to duplicate keys and return the result. If
+    there are colliding keys with non-MappingNode values (e.g., integers or strings), the last
+    of the values wins.
+
+    For instance, given node values of:
+
+        [
+            (
+                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                MappingNode(tag='tag:yaml.org,2002:map', value=[
+                    (
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                    ),
+                    (
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='7')
+                    ),
+                ]),
+            ),
+            (
+                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                MappingNode(tag='tag:yaml.org,2002:map', value=[
+                    (
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='5')
+                    ),
+                ]),
+            ),
+        ]
+
+    ... the returned result would be:
+
+        [
+            (
+                ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+                MappingNode(tag='tag:yaml.org,2002:map', value=[
+                    (
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_hourly'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='24')
+                    ),
+                    (
+                        ScalarNode(tag='tag:yaml.org,2002:str', value='keep_daily'),
+                        ScalarNode(tag='tag:yaml.org,2002:int', value='5')
+                    ),
+                ]),
+            ),
+        ]
+
+    The purpose of deep merging like this is to support, for instance, merging one borgmatic
+    configuration file into another for reuse, such that a configuration section ("retention",
+    etc.) does not completely replace the corresponding section in a merged file.
+    '''
+    # Map from original node key/value to the replacement merged node. DELETED_NODE as a replacement
+    # node indications deletion.
+    replaced_nodes = {}
+
+    # To find nodes that require merging, compare each node with each other node.
+    for a_key, a_value in nodes:
+        for b_key, b_value in nodes:
+            # If we've already considered one of the nodes for merging, skip it.
+            if (a_key, a_value) in replaced_nodes or (b_key, b_value) in replaced_nodes:
+                continue
+
+            # If the keys match and the values are different, we need to merge these two A and B nodes.
+            if a_key.tag == b_key.tag and a_key.value == b_key.value and a_value != b_value:
+                # Since we're merging into the B node, consider the A node a duplicate and remove it.
+                replaced_nodes[(a_key, a_value)] = DELETED_NODE
+
+                # If we're dealing with MappingNodes, recurse and merge its values as well.
+                if isinstance(b_value, ruamel.yaml.nodes.MappingNode):
+                    replaced_nodes[(b_key, b_value)] = (
+                        b_key,
+                        ruamel.yaml.nodes.MappingNode(
+                            tag=b_value.tag,
+                            value=deep_merge_nodes(a_value.value + b_value.value),
+                            start_mark=b_value.start_mark,
+                            end_mark=b_value.end_mark,
+                            flow_style=b_value.flow_style,
+                            comment=b_value.comment,
+                            anchor=b_value.anchor,
+                        ),
+                    )
+
+    return [
+        replaced_nodes.get(node, node) for node in nodes if replaced_nodes.get(node) != DELETED_NODE
+    ]
+
+
 class Include_constructor(ruamel.yaml.SafeConstructor):
     '''
     A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
@@ -40,14 +141,19 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
 
     def flatten_mapping(self, node):
         '''
-        Support the special case of shallow merging included configuration into an existing mapping
+        Support the special case of deep merging included configuration into an existing mapping
         using the YAML '<<' merge key. Example syntax:
 
         ```
         retention:
             keep_daily: 1
-            <<: !include common.yaml
+
+        <<: !include common.yaml
         ```
+
+        These includes are deep merged into the current configuration file. For instance, in this
+        example, any "retention" options in common.yaml will get merged into the "retention" section
+        in the example configuration file.
         '''
         representer = ruamel.yaml.representer.SafeRepresenter()
 
@@ -57,3 +163,5 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
                 node.value[index] = (key_node, included_value)
 
         super(Include_constructor, self).flatten_mapping(node)
+
+        node.value = deep_merge_nodes(node.value)

borgmatic/config/schema.yaml

@@ -42,13 +42,21 @@ properties:
                 example:
                     - user@backupserver:sourcehostname.borg
                     - "user@backupserver:{fqdn}"
+            working_directory:
+                type: string
+                description: |
+                    Working directory for the "borg create" command. Tildes are
+                    expanded. Useful for backing up using relative paths. See
+                    http://borgbackup.readthedocs.io/en/stable/usage/create.html
+                    for details. Defaults to not set.
+                example: /path/to/working/directory
             one_file_system:
                 type: boolean
                 description: |
-                    Stay in same file system (do not cross mount points).
-                    Defaults to false. But when a database hook is used, the
-                    setting here is ignored and one_file_system is considered
-                    true.
+                    Stay in same file system: do not cross mount points beyond
+                    the given source directories. Defaults to false. But when a
+                    database hook is used, the setting here is ignored and
+                    one_file_system is considered true.
                 example: true
             numeric_owner:
                 type: boolean

borgmatic/hooks/healthchecks.py

@@ -13,7 +13,7 @@ MONITOR_STATE_TO_HEALTHCHECKS = {
 }
 
 PAYLOAD_TRUNCATION_INDICATOR = '...\n'
-PAYLOAD_LIMIT_BYTES = 10 * 1024 - len(PAYLOAD_TRUNCATION_INDICATOR)
+PAYLOAD_LIMIT_BYTES = 100 * 1024 - len(PAYLOAD_TRUNCATION_INDICATOR)
 
 
 class Forgetful_buffering_handler(logging.Handler):

borgmatic/signals.py

@@ -1,23 +1,34 @@
+import logging
 import os
 import signal
+import sys
+
+logger = logging.getLogger(__name__)
 
 
-def _handle_signal(signal_number, frame):  # pragma: no cover
+EXIT_CODE_FROM_SIGNAL = 128
+
+
+def handle_signal(signal_number, frame):
     '''
     Send the signal to all processes in borgmatic's process group, which includes child processes.
     '''
     # Prevent infinite signal handler recursion. If the parent frame is this very same handler
     # function, we know we're recursing.
-    if frame.f_back.f_code.co_name == _handle_signal.__name__:
+    if frame.f_back.f_code.co_name == handle_signal.__name__:
         return
 
     os.killpg(os.getpgrp(), signal_number)
 
+    if signal_number == signal.SIGTERM:
+        logger.critical('Exiting due to TERM signal')
+        sys.exit(EXIT_CODE_FROM_SIGNAL + signal.SIGTERM)
 
-def configure_signals():  # pragma: no cover
+
+def configure_signals():
     '''
     Configure borgmatic's signal handlers to pass relevant signals through to any child processes
     like Borg. Note that SIGINT gets passed through even without these changes.
     '''
     for signal_number in (signal.SIGHUP, signal.SIGTERM, signal.SIGUSR1, signal.SIGUSR2):
-        signal.signal(signal_number, _handle_signal)
+        signal.signal(signal_number, handle_signal)

docs/_includes/index.css

@@ -258,6 +258,7 @@ footer.elv-layout {
 /* Header */
 .elv-header {
 	position: relative;
+        text-align: center;
 }
 .elv-header-default {
 	display: flex;

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -7,11 +7,12 @@ eleventyNavigation:
 ---
 ## Preparation and cleanup hooks
 
-If you find yourself performing prepraration tasks before your backup runs, or
+If you find yourself performing preparation tasks before your backup runs, or
 cleanup work afterwards, borgmatic hooks may be of interest. Hooks are shell
-commands that borgmatic executes for you at various points, and they're
-configured in the `hooks` section of your configuration file. But if you're
-looking to backup a database, it's probably easier to use the [database backup
+commands that borgmatic executes for you at various points as it runs, and
+they're configured in the `hooks` section of your configuration file. But if
+you're looking to backup a database, it's probably easier to use the [database
+backup
 feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
 instead.
 
@@ -27,15 +28,40 @@ hooks:
         - umount /some/filesystem
 ```
 
-The `before_backup` and `after_backup` hooks each run once per configuration
-file. `before_backup` hooks run prior to backups of all repositories in a
-configuration file, right before the `create` action. `after_backup` hooks run
-afterwards, but not if an error occurs in a previous hook or in the backups
-themselves.
+The `before_backup` and `after_backup` hooks each run once per repository in a
+configuration file. `before_backup` hooks runs right before the `create`
+action for a particular repository, and `after_backup` hooks run afterwards,
+but not if an error occurs in a previous hook or in the backups themselves.
+(Prior to borgmatic 1.6.0, these hooks instead ran once per configuration file
+rather than once per repository.)
 
 There are additional hooks that run before/after other actions as well. For
-instance, `before_prune` runs before a `prune` action, while `after_prune`
-runs after it.
+instance, `before_prune` runs before a `prune` action for a repository, while
+`after_prune` runs after it.
+
+## Variable interpolation
+
+The before and after action hooks support interpolating particular runtime
+variables into the hook command. Here's an example that assumes you provide a
+separate shell script:
+
+```yaml
+hooks:
+    after_prune:
+        - record-prune.sh "{configuration_filename}" "{repository}"
+```
+
+In this example, when the hook is triggered, borgmatic interpolates runtime
+values into the hook command: the borgmatic configuration filename and the
+paths of the current Borg repository. Here's the full set of supported
+variables you can use here:
+
+ * `configuration_filename`: borgmatic configuration filename in which the
+   hook was defined
+ * `repository`: path of the current repository as configured in the current
+   borgmatic configuration file
+
+## Global hooks
 
 You can also use `before_everything` and `after_everything` hooks to perform
 global setup or cleanup:
@@ -58,6 +84,8 @@ but only if there is a `create` action. It runs even if an error occurs during
 a backup or a backup hook, but not if an error occurs during a
 `before_everything` hook.
 
+## Error hooks
+
 borgmatic also runs `on_error` hooks if an error occurs, either when creating
 a backup or running a backup hook. See the [monitoring and alerting
 documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)

docs/how-to/backup-your-databases.md

@@ -199,10 +199,10 @@ backups to avoid getting caught without a way to restore a database.
 databases that share the exact same name on different hosts.
 4. Because database hooks implicitly enable the `read_special` configuration
 setting to support dump and restore streaming, you'll need to ensure that any
-special files are excluded from backups (named pipes, block devices, and
-character devices) to prevent hanging. Try a command like `find / -type c,b,p`
-to find such files. Common directories to exclude are `/dev` and `/run`, but
-that may not be exhaustive.
+special files are excluded from backups (named pipes, block devices,
+character devices, and sockets) to prevent hanging. Try a command like
+`find /your/source/path -type c,b,p,s` to find such files. Common directories
+to exclude are `/dev` and `/run`, but that may not be exhaustive.
 
 
 ### Manual restoration
@@ -244,5 +244,10 @@ hooks:
 ### borgmatic hangs during backup
 
 See Limitations above about `read_special`. You may need to exclude certain
-paths with named pipes, block devices, or character devices on which borgmatic
-is hanging.
+paths with named pipes, block devices, character devices, or sockets on which
+borgmatic is hanging.
+
+Alternatively, if excluding special files is too onerous, you can create two
+separate borgmatic configuration files—one for your source files and a
+separate one for backing up databases. That way, the database `read_special`
+option will not be active when backing up special files.

docs/how-to/make-per-application-backups.md

@@ -80,7 +80,7 @@ location:
 !include /etc/borgmatic/common_retention.yaml
 ```
 
-But if you do want to merge in a YAML key and its values, keep reading!
+But if you do want to merge in a YAML key *and* its values, keep reading!
 
 
 ## Include merging
@@ -89,31 +89,35 @@ If you need to get even fancier and pull in common configuration options while
 potentially overriding individual options, you can perform a YAML merge of
 included configuration using the YAML `<<` key. For instance, here's an
 example of a main configuration file that pulls in two retention options via
-an include, and then overrides one of them locally:
+an include and then overrides one of them locally:
 
 ```yaml
+<<: !include /etc/borgmatic/common.yaml
+
 location:
    ...
 
 retention:
     keep_daily: 5
-    <<: !include /etc/borgmatic/common_retention.yaml
 ```
 
-This is what `common_retention.yaml` might look like:
+This is what `common.yaml` might look like:
 
 ```yaml
-keep_hourly: 24
-keep_daily: 7
+retention:
+    keep_hourly: 24
+    keep_daily: 7
 ```
 
 Once this include gets merged in, the resulting configuration would have a
 `keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
 
-When there is a collision of an option between the local file and the merged
-include, the local file's option takes precedent. And note that this is a
-shallow merge rather than a deep merge, so the merging does not descend into
-nested values.
+When there's an option collision between the local file and the merged
+include, the local file's option takes precedent. And as of borgmatic 1.6.0,
+this feature performs a deep merge, meaning that values are merged at all
+levels in the two configuration files. This allows you to include common
+configuration—up to full borgmatic configuration files—while overriding only
+the parts you want to customize.
 
 Note that this `<<` include merging syntax is only for merging in mappings
 (keys/values). If you'd like to include other types like scalars or lists

docs/how-to/monitor-your-backups.md

@@ -104,10 +104,9 @@ hooks:
         - send-text-message.sh "{configuration_filename}" "{repository}"
 ```
 
-In this example, when the error occurs, borgmatic interpolates a few runtime
-values into the hook command: the borgmatic configuration filename, and the
-path of the repository. Here's the full set of supported variables you can use
-here:
+In this example, when the error occurs, borgmatic interpolates runtime values
+into the hook command: the borgmatic configuration filename, and the path of
+the repository. Here's the full set of supported variables you can use here:
 
  * `configuration_filename`: borgmatic configuration filename in which the
    error occurred

docs/how-to/set-up-backups.md

@@ -28,7 +28,7 @@ sudo pip3 install --user --upgrade borgmatic
 This installs borgmatic and its commands at the `/root/.local/bin` path.
 
 Your pip binary may have a different name than "pip3". Make sure you're using
-Python 3.6+, as borgmatic does not support Python 2.
+Python 3.7+, as borgmatic does not support older versions of Python.
 
 The next step is to ensure that borgmatic's commands available are on your
 system `PATH`, so that you can run borgmatic:
@@ -51,6 +51,11 @@ sudo borgmatic --version
 
 If borgmatic is properly installed, that should output your borgmatic version.
 
+As an alternative to adding the path to `~/.bashrc` file, if you're using sudo
+to run borgmatic, you can configure [sudo's
+`secure_path` option](https://man.archlinux.org/man/sudoers.5) to include
+borgmatic's path.
+
 
 ### Global install option
 

scripts/release

@@ -31,8 +31,8 @@ python3 setup.py bdist_wheel
 python3 setup.py sdist
 gpg --detach-sign --armor dist/borgmatic-*.tar.gz
 gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl
-twine upload -r pypi dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
-twine upload -r pypi dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none-any.whl.asc
+twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
+twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none-any.whl.asc
 
 # Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.23'
+VERSION = '1.6.0'
 
 
 setup(

tests/integration/config/test_load.py

@@ -60,3 +60,184 @@ def test_load_configuration_does_not_merge_include_list():
 
     with pytest.raises(ruamel.yaml.error.YAMLError):
         assert module.load_configuration('config.yaml')
+
+
+def test_deep_merge_nodes_replaces_colliding_scalar_values():
+    node_values = [
+        (
+            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_hourly'
+                        ),
+                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
+                    ),
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                    ),
+                ],
+            ),
+        ),
+        (
+            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'retention'
+    options = section_value.value
+    assert len(options) == 2
+    assert options[0][0].value == 'keep_hourly'
+    assert options[0][1].value == '24'
+    assert options[1][0].value == 'keep_daily'
+    assert options[1][1].value == '5'
+
+
+def test_deep_merge_nodes_keeps_non_colliding_scalar_values():
+    node_values = [
+        (
+            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_hourly'
+                        ),
+                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='24'),
+                    ),
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_daily'
+                        ),
+                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='7'),
+                    ),
+                ],
+            ),
+        ),
+        (
+            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='retention'),
+            ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='keep_minutely'
+                        ),
+                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='10'),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'retention'
+    options = section_value.value
+    assert len(options) == 3
+    assert options[0][0].value == 'keep_hourly'
+    assert options[0][1].value == '24'
+    assert options[1][0].value == 'keep_daily'
+    assert options[1][1].value == '7'
+    assert options[2][0].value == 'keep_minutely'
+    assert options[2][1].value == '10'
+
+
+def test_deep_merge_nodes_keeps_deeply_nested_values():
+    node_values = [
+        (
+            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
+            ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='lock_wait'
+                        ),
+                        ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:int', value='5'),
+                    ),
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='extra_borg_options'
+                        ),
+                        ruamel.yaml.nodes.MappingNode(
+                            tag='tag:yaml.org,2002:map',
+                            value=[
+                                (
+                                    ruamel.yaml.nodes.ScalarNode(
+                                        tag='tag:yaml.org,2002:str', value='init'
+                                    ),
+                                    ruamel.yaml.nodes.ScalarNode(
+                                        tag='tag:yaml.org,2002:str', value='--init-option'
+                                    ),
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+        (
+            ruamel.yaml.nodes.ScalarNode(tag='tag:yaml.org,2002:str', value='storage'),
+            ruamel.yaml.nodes.MappingNode(
+                tag='tag:yaml.org,2002:map',
+                value=[
+                    (
+                        ruamel.yaml.nodes.ScalarNode(
+                            tag='tag:yaml.org,2002:str', value='extra_borg_options'
+                        ),
+                        ruamel.yaml.nodes.MappingNode(
+                            tag='tag:yaml.org,2002:map',
+                            value=[
+                                (
+                                    ruamel.yaml.nodes.ScalarNode(
+                                        tag='tag:yaml.org,2002:str', value='prune'
+                                    ),
+                                    ruamel.yaml.nodes.ScalarNode(
+                                        tag='tag:yaml.org,2002:str', value='--prune-option'
+                                    ),
+                                ),
+                            ],
+                        ),
+                    ),
+                ],
+            ),
+        ),
+    ]
+
+    result = module.deep_merge_nodes(node_values)
+    assert len(result) == 1
+    (section_key, section_value) = result[0]
+    assert section_key.value == 'storage'
+    options = section_value.value
+    assert len(options) == 2
+    assert options[0][0].value == 'lock_wait'
+    assert options[0][1].value == '5'
+    assert options[1][0].value == 'extra_borg_options'
+    nested_options = options[1][1].value
+    assert len(nested_options) == 2
+    assert nested_options[0][0].value == 'init'
+    assert nested_options[0][1].value == '--init-option'
+    assert nested_options[1][0].value == 'prune'
+    assert nested_options[1][1].value == '--prune-option'

tests/unit/borg/test_borg.py

@@ -121,3 +121,13 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
     module.run_arbitrary_borg(
         repository='repo', storage_config={}, options=[],
     )
+
+
+def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'key', 'export', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['key', 'export'],
+    )

tests/unit/borg/test_compact.py

@@ -36,8 +36,8 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter():
     module.compact_segments(repository='repo', storage_config={}, dry_run=False)
 
 
-def test_compact_segments_with_dry_run_calls_borg_with_dry_run_parameter():
-    insert_execute_command_mock(COMPACT_COMMAND + ('--dry-run', 'repo'), logging.INFO)
+def test_compact_segments_with_dry_run_skips_borg_call():
+    flexmock(module).should_receive('execute_command').never()
 
     module.compact_segments(repository='repo', storage_config={}, dry_run=True)
 

tests/unit/commands/test_borgmatic.py

@@ -35,99 +35,14 @@ def test_run_configuration_with_invalid_borg_version_errors():
     list(module.run_configuration('test.yaml', config, arguments))
 
 
-def test_run_configuration_calls_hooks_for_prune_action():
+def test_run_configuration_logs_monitor_start_error():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').twice()
-    flexmock(module.dispatch).should_receive('call_hooks').at_least().twice()
-    flexmock(module).should_receive('run_actions').and_return([])
-    config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()}
-
-    list(module.run_configuration('test.yaml', config, arguments))
-
-
-def test_run_configuration_calls_hooks_for_compact_action():
-    flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').twice()
-    flexmock(module).should_receive('run_actions').and_return([])
-    config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'compact': flexmock()}
-
-    list(module.run_configuration('test.yaml', config, arguments))
-
-
-def test_run_configuration_executes_and_calls_hooks_for_create_action():
-    flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').twice()
-    flexmock(module.dispatch).should_receive('call_hooks').at_least().twice()
-    flexmock(module).should_receive('run_actions').and_return([])
-    config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
-
-    list(module.run_configuration('test.yaml', config, arguments))
-
-
-def test_run_configuration_calls_hooks_for_check_action():
-    flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').twice()
-    flexmock(module.dispatch).should_receive('call_hooks').at_least().twice()
-    flexmock(module).should_receive('run_actions').and_return([])
-    config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'check': flexmock()}
-
-    list(module.run_configuration('test.yaml', config, arguments))
-
-
-def test_run_configuration_calls_hooks_for_extract_action():
-    flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').twice()
-    flexmock(module.dispatch).should_receive('call_hooks').never()
-    flexmock(module).should_receive('run_actions').and_return([])
-    config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'extract': flexmock()}
-
-    list(module.run_configuration('test.yaml', config, arguments))
-
-
-def test_run_configuration_does_not_trigger_hooks_for_list_action():
-    flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').never()
-    flexmock(module.dispatch).should_receive('call_hooks').never()
-    flexmock(module).should_receive('run_actions').and_return([])
-    config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'list': flexmock()}
-
-    list(module.run_configuration('test.yaml', config, arguments))
-
-
-def test_run_configuration_logs_actions_error():
-    flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook')
-    flexmock(module.dispatch).should_receive('call_hooks')
+    flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return(
+        None
+    ).and_return(None)
     expected_results = [flexmock()]
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
-    flexmock(module).should_receive('run_actions').and_raise(OSError)
-    config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)}
-
-    results = list(module.run_configuration('test.yaml', config, arguments))
-
-    assert results == expected_results
-
-
-def test_run_configuration_logs_pre_hook_error():
-    flexmock(module.borg_environment).should_receive('initialize')
-    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').and_raise(OSError).and_return(None)
-    expected_results = [flexmock()]
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
+    flexmock(module).should_receive('log_error_records').and_return(expected_results)
     flexmock(module).should_receive('run_actions').never()
     config = {'location': {'repositories': ['foo']}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
@@ -137,12 +52,12 @@ def test_run_configuration_logs_pre_hook_error():
     assert results == expected_results
 
 
-def test_run_configuration_bails_for_pre_hook_soft_failure():
+def test_run_configuration_bails_for_monitor_start_soft_failure():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
-    flexmock(module.command).should_receive('execute_hook').and_raise(error).and_return(None)
-    flexmock(module).should_receive('make_error_log_records').never()
+    flexmock(module.dispatch).should_receive('call_hooks').and_raise(error)
+    flexmock(module).should_receive('log_error_records').never()
     flexmock(module).should_receive('run_actions').never()
     config = {'location': {'repositories': ['foo']}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
@@ -152,15 +67,46 @@ def test_run_configuration_bails_for_pre_hook_soft_failure():
     assert results == []
 
 
-def test_run_configuration_logs_post_hook_error():
+def test_run_configuration_logs_actions_error():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
-    flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
-        OSError
-    ).and_return(None)
+    flexmock(module.command).should_receive('execute_hook')
     flexmock(module.dispatch).should_receive('call_hooks')
     expected_results = [flexmock()]
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
+    flexmock(module).should_receive('log_error_records').and_return(expected_results)
+    flexmock(module).should_receive('run_actions').and_raise(OSError)
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == expected_results
+
+
+def test_run_configuration_bails_for_actions_soft_failure():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
+    flexmock(module.dispatch).should_receive('call_hooks')
+    error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
+    flexmock(module).should_receive('run_actions').and_raise(error)
+    flexmock(module).should_receive('log_error_records').never()
+    flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == []
+
+
+def test_run_configuration_logs_monitor_finish_error():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
+    flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
+        None
+    ).and_raise(OSError)
+    expected_results = [flexmock()]
+    flexmock(module).should_receive('log_error_records').and_return(expected_results)
     flexmock(module).should_receive('run_actions').and_return([])
     config = {'location': {'repositories': ['foo']}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
@@ -170,16 +116,16 @@ def test_run_configuration_logs_post_hook_error():
     assert results == expected_results
 
 
-def test_run_configuration_bails_for_post_hook_soft_failure():
+def test_run_configuration_bails_for_monitor_finish_soft_failure():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
-    flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
-        error
-    ).and_return(None)
-    flexmock(module.dispatch).should_receive('call_hooks')
-    flexmock(module).should_receive('make_error_log_records').never()
+    flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
+        None
+    ).and_raise(error)
+    flexmock(module).should_receive('log_error_records').never()
     flexmock(module).should_receive('run_actions').and_return([])
+    flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
     config = {'location': {'repositories': ['foo']}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
 
@@ -193,7 +139,7 @@ def test_run_configuration_logs_on_error_hook_error():
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
     expected_results = [flexmock(), flexmock()]
-    flexmock(module).should_receive('make_error_log_records').and_return(
+    flexmock(module).should_receive('log_error_records').and_return(
         expected_results[:1]
     ).and_return(expected_results[1:])
     flexmock(module).should_receive('run_actions').and_raise(OSError)
@@ -209,9 +155,9 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
-    flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(error)
+    flexmock(module.command).should_receive('execute_hook').and_raise(error)
     expected_results = [flexmock()]
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
+    flexmock(module).should_receive('log_error_records').and_return(expected_results)
     flexmock(module).should_receive('run_actions').and_raise(OSError)
     config = {'location': {'repositories': ['foo']}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
@@ -227,12 +173,11 @@ def test_run_configuration_retries_soft_error():
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([])
-    expected_results = [flexmock()]
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_results).once()
+    flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once()
     config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
     results = list(module.run_configuration('test.yaml', config, arguments))
-    assert results == expected_results
+    assert results == []
 
 
 def test_run_configuration_retries_hard_error():
@@ -241,18 +186,20 @@ def test_run_configuration_retries_hard_error():
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
-    expected_results = [flexmock(), flexmock()]
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[:1]).with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(
-        expected_results[1:]
-    ).twice()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()])
+    error_logs = [flexmock()]
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository', OSError,
+    ).and_return(error_logs)
     config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
     results = list(module.run_configuration('test.yaml', config, arguments))
-    assert results == expected_results
+    assert results == error_logs
 
 
 def test_run_repos_ordered():
@@ -261,10 +208,10 @@ def test_run_repos_ordered():
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
     expected_results = [flexmock(), flexmock()]
-    flexmock(module).should_receive('make_error_log_records').with_args(
+    flexmock(module).should_receive('log_error_records').with_args(
         'foo: Error running actions for repository', OSError
     ).and_return(expected_results[:1]).ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
+    flexmock(module).should_receive('log_error_records').with_args(
         'bar: Error running actions for repository', OSError
     ).and_return(expected_results[1:]).ordered()
     config = {'location': {'repositories': ['foo', 'bar']}}
@@ -278,23 +225,30 @@ def test_run_configuration_retries_round_robbin():
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
-    expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
-    flexmock(module).should_receive('make_error_log_records').with_args(
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'bar: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
+    foo_error_logs = [flexmock()]
+    flexmock(module).should_receive('log_error_records').with_args(
         'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[0:1]).ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
+    ).and_return(foo_error_logs).ordered()
+    bar_error_logs = [flexmock()]
+    flexmock(module).should_receive('log_error_records').with_args(
         'bar: Error running actions for repository', OSError
-    ).and_return(expected_results[1:2]).ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[2:3]).ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'bar: Error running actions for repository', OSError
-    ).and_return(expected_results[3:4]).ordered()
+    ).and_return(bar_error_logs).ordered()
     config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
     results = list(module.run_configuration('test.yaml', config, arguments))
-    assert results == expected_results
+    assert results == foo_error_logs + bar_error_logs
 
 
 def test_run_configuration_retries_one_passes():
@@ -304,20 +258,26 @@ def test_run_configuration_retries_one_passes():
     flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
         []
     ).and_raise(OSError).times(4)
-    expected_results = [flexmock(), flexmock(), flexmock()]
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[0:1]).ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'bar: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return(flexmock()).ordered()
+    error_logs = [flexmock()]
+    flexmock(module).should_receive('log_error_records').with_args(
         'bar: Error running actions for repository', OSError
-    ).and_return(expected_results[1:2]).ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'bar: Error running actions for repository', OSError
-    ).and_return(expected_results[2:3]).ordered()
+    ).and_return(error_logs).ordered()
     config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
     results = list(module.run_configuration('test.yaml', config, arguments))
-    assert results == expected_results
+    assert results == error_logs
 
 
 def test_run_configuration_retry_wait():
@@ -325,29 +285,38 @@ def test_run_configuration_retry_wait():
     flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
     flexmock(module.command).should_receive('execute_hook')
     flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
-    expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[0:1]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
 
     flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[1:2]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
 
     flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[2:3]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
 
     flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
+    error_logs = [flexmock()]
+    flexmock(module).should_receive('log_error_records').with_args(
         'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[3:4]).ordered()
+    ).and_return(error_logs).ordered()
     config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}}
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
     results = list(module.run_configuration('test.yaml', config, arguments))
-    assert results == expected_results
+    assert results == error_logs
 
 
 def test_run_configuration_retries_timeout_multiple_repos():
@@ -357,29 +326,342 @@ def test_run_configuration_retries_timeout_multiple_repos():
     flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
         []
     ).and_raise(OSError).times(4)
-    expected_results = [flexmock(), flexmock(), flexmock()]
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'foo: Error running actions for repository', OSError
-    ).and_return(expected_results[0:1]).ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
-        'bar: Error running actions for repository', OSError
-    ).and_return(expected_results[1:2]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'foo: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
+    flexmock(module).should_receive('log_error_records').with_args(
+        'bar: Error running actions for repository',
+        OSError,
+        levelno=logging.WARNING,
+        log_command_error_output=True,
+    ).and_return([flexmock()]).ordered()
 
     # Sleep before retrying foo (and passing)
     flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
 
     # Sleep before retrying bar (and failing)
     flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
-    flexmock(module).should_receive('make_error_log_records').with_args(
+    error_logs = [flexmock()]
+    flexmock(module).should_receive('log_error_records').with_args(
         'bar: Error running actions for repository', OSError
-    ).and_return(expected_results[2:3]).ordered()
+    ).and_return(error_logs).ordered()
     config = {
         'location': {'repositories': ['foo', 'bar']},
         'storage': {'retries': 1, 'retry_wait': 10},
     }
     arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
     results = list(module.run_configuration('test.yaml', config, arguments))
-    assert results == expected_results
+    assert results == error_logs
+
+
+def test_run_actions_does_not_raise_for_init_action():
+    flexmock(module.borg_init).should_receive('initialize_repository')
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'init': flexmock(
+            encryption_mode=flexmock(), append_only=flexmock(), storage_quota=flexmock()
+        ),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_calls_hooks_for_prune_action():
+    flexmock(module.borg_prune).should_receive('prune_archives')
+    flexmock(module.command).should_receive('execute_hook').twice()
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'prune': flexmock(stats=flexmock(), files=flexmock()),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_calls_hooks_for_compact_action():
+    flexmock(module.borg_feature).should_receive('available').and_return(True)
+    flexmock(module.borg_compact).should_receive('compact_segments')
+    flexmock(module.command).should_receive('execute_hook').twice()
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'compact': flexmock(progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock()),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_executes_and_calls_hooks_for_create_action():
+    flexmock(module.borg_create).should_receive('create_archive')
+    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module.dispatch).should_receive('call_hooks').and_return({}).times(3)
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'create': flexmock(
+            progress=flexmock(), stats=flexmock(), json=flexmock(), files=flexmock()
+        ),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_calls_hooks_for_check_action():
+    flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True)
+    flexmock(module.borg_check).should_receive('check_archives')
+    flexmock(module.command).should_receive('execute_hook').twice()
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'check': flexmock(progress=flexmock(), repair=flexmock(), only=flexmock()),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_calls_hooks_for_extract_action():
+    flexmock(module.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borg_extract).should_receive('extract_archive')
+    flexmock(module.command).should_receive('execute_hook').twice()
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'extract': flexmock(
+            paths=flexmock(),
+            progress=flexmock(),
+            destination=flexmock(),
+            strip_components=flexmock(),
+            archive=flexmock(),
+            repository='repo',
+        ),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_does_not_raise_for_export_tar_action():
+    flexmock(module.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borg_export_tar).should_receive('export_tar_archive')
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'export-tar': flexmock(
+            repository=flexmock(),
+            archive=flexmock(),
+            paths=flexmock(),
+            destination=flexmock(),
+            tar_filter=flexmock(),
+            files=flexmock(),
+            strip_components=flexmock(),
+        ),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_does_not_raise_for_mount_action():
+    flexmock(module.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borg_mount).should_receive('mount_archive')
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'mount': flexmock(
+            repository=flexmock(),
+            archive=flexmock(),
+            mount_point=flexmock(),
+            paths=flexmock(),
+            foreground=flexmock(),
+            options=flexmock(),
+        ),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_does_not_raise_for_list_action():
+    flexmock(module.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
+    flexmock(module.borg_list).should_receive('list_archives')
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'list': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_does_not_raise_for_info_action():
+    flexmock(module.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
+    flexmock(module.borg_info).should_receive('display_archives_info')
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'info': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
+def test_run_actions_does_not_raise_for_borg_action():
+    flexmock(module.validate).should_receive('repositories_match').and_return(True)
+    flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
+    flexmock(module.borg_borg).should_receive('run_arbitrary_borg')
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'borg': flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
 
 
 def test_load_configurations_collects_parsed_configurations():
@@ -395,6 +677,15 @@ def test_load_configurations_collects_parsed_configurations():
     assert logs == []
 
 
+def test_load_configurations_logs_warning_for_permission_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError)
+
+    configs, logs = tuple(module.load_configurations(('test.yaml',)))
+
+    assert configs == {}
+    assert {log.levelno for log in logs} == {logging.WARNING}
+
+
 def test_load_configurations_logs_critical_for_parse_error():
     flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
 
@@ -412,48 +703,46 @@ def test_log_record_with_suppress_does_not_raise():
     module.log_record(levelno=1, foo='bar', baz='quux', suppress_log=True)
 
 
-def test_make_error_log_records_generates_output_logs_for_message_only():
+def test_log_error_records_generates_output_logs_for_message_only():
     flexmock(module).should_receive('log_record').replace_with(dict)
 
-    logs = tuple(module.make_error_log_records('Error'))
+    logs = tuple(module.log_error_records('Error'))
 
     assert {log['levelno'] for log in logs} == {logging.CRITICAL}
 
 
-def test_make_error_log_records_generates_output_logs_for_called_process_error():
+def test_log_error_records_generates_output_logs_for_called_process_error():
     flexmock(module).should_receive('log_record').replace_with(dict)
     flexmock(module.logger).should_receive('getEffectiveLevel').and_return(logging.WARNING)
 
     logs = tuple(
-        module.make_error_log_records(
-            'Error', subprocess.CalledProcessError(1, 'ls', 'error output')
-        )
+        module.log_error_records('Error', subprocess.CalledProcessError(1, 'ls', 'error output'))
     )
 
     assert {log['levelno'] for log in logs} == {logging.CRITICAL}
     assert any(log for log in logs if 'error output' in str(log))
 
 
-def test_make_error_log_records_generates_logs_for_value_error():
+def test_log_error_records_generates_logs_for_value_error():
     flexmock(module).should_receive('log_record').replace_with(dict)
 
-    logs = tuple(module.make_error_log_records('Error', ValueError()))
+    logs = tuple(module.log_error_records('Error', ValueError()))
 
     assert {log['levelno'] for log in logs} == {logging.CRITICAL}
 
 
-def test_make_error_log_records_generates_logs_for_os_error():
+def test_log_error_records_generates_logs_for_os_error():
     flexmock(module).should_receive('log_record').replace_with(dict)
 
-    logs = tuple(module.make_error_log_records('Error', OSError()))
+    logs = tuple(module.log_error_records('Error', OSError()))
 
     assert {log['levelno'] for log in logs} == {logging.CRITICAL}
 
 
-def test_make_error_log_records_generates_nothing_for_other_error():
+def test_log_error_records_generates_nothing_for_other_error():
     flexmock(module).should_receive('log_record').replace_with(dict)
 
-    logs = tuple(module.make_error_log_records('Error', KeyError()))
+    logs = tuple(module.log_error_records('Error', KeyError()))
 
     assert logs == ()
 
@@ -510,7 +799,7 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error():
         ValueError
     )
     expected_logs = (flexmock(),)
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    flexmock(module).should_receive('log_error_records').and_return(expected_logs)
     arguments = {'extract': flexmock(repository='repo')}
 
     logs = tuple(
@@ -537,7 +826,7 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error():
         ValueError
     )
     expected_logs = (flexmock(),)
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    flexmock(module).should_receive('log_error_records').and_return(expected_logs)
     arguments = {'mount': flexmock(repository='repo')}
 
     logs = tuple(
@@ -550,7 +839,7 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error():
 def test_collect_configuration_run_summary_logs_missing_configs_error():
     arguments = {'global': flexmock(config_paths=[])}
     expected_logs = (flexmock(),)
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    flexmock(module).should_receive('log_error_records').and_return(expected_logs)
 
     logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
 
@@ -560,7 +849,7 @@ def test_collect_configuration_run_summary_logs_missing_configs_error():
 def test_collect_configuration_run_summary_logs_pre_hook_error():
     flexmock(module.command).should_receive('execute_hook').and_raise(ValueError)
     expected_logs = (flexmock(),)
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    flexmock(module).should_receive('log_error_records').and_return(expected_logs)
     arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)}
 
     logs = tuple(
@@ -574,7 +863,7 @@ def test_collect_configuration_run_summary_logs_post_hook_error():
     flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(ValueError)
     flexmock(module).should_receive('run_configuration').and_return([])
     expected_logs = (flexmock(),)
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    flexmock(module).should_receive('log_error_records').and_return(expected_logs)
     arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)}
 
     logs = tuple(
@@ -589,7 +878,7 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi
         ValueError
     )
     expected_logs = (flexmock(),)
-    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    flexmock(module).should_receive('log_error_records').and_return(expected_logs)
     arguments = {'list': flexmock(repository='repo', archive='test')}
 
     logs = tuple(
@@ -615,7 +904,7 @@ def test_collect_configuration_run_summary_logs_run_configuration_error():
     flexmock(module).should_receive('run_configuration').and_return(
         [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))]
     )
-    flexmock(module).should_receive('make_error_log_records').and_return([])
+    flexmock(module).should_receive('log_error_records').and_return([])
     arguments = {}
 
     logs = tuple(
@@ -629,7 +918,7 @@ def test_collect_configuration_run_summary_logs_run_umount_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
     flexmock(module).should_receive('run_configuration').and_return([])
     flexmock(module.borg_umount).should_receive('unmount_archive').and_raise(OSError)
-    flexmock(module).should_receive('make_error_log_records').and_return(
+    flexmock(module).should_receive('log_error_records').and_return(
         [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))]
     )
     arguments = {'umount': flexmock(mount_point='/mnt')}

tests/unit/test_signals.py

@@ -0,0 +1,40 @@
+from flexmock import flexmock
+
+from borgmatic import signals as module
+
+
+def test_handle_signal_forwards_to_subprocesses():
+    signal_number = 100
+    frame = flexmock(f_back=flexmock(f_code=flexmock(co_name='something')))
+    process_group = flexmock()
+    flexmock(module.os).should_receive('getpgrp').and_return(process_group)
+    flexmock(module.os).should_receive('killpg').with_args(process_group, signal_number).once()
+
+    module.handle_signal(signal_number, frame)
+
+
+def test_handle_signal_bails_on_recursion():
+    signal_number = 100
+    frame = flexmock(f_back=flexmock(f_code=flexmock(co_name='handle_signal')))
+    flexmock(module.os).should_receive('getpgrp').never()
+    flexmock(module.os).should_receive('killpg').never()
+
+    module.handle_signal(signal_number, frame)
+
+
+def test_handle_signal_exits_on_sigterm():
+    signal_number = module.signal.SIGTERM
+    frame = flexmock(f_back=flexmock(f_code=flexmock(co_name='something')))
+    flexmock(module.os).should_receive('getpgrp').and_return(flexmock)
+    flexmock(module.os).should_receive('killpg')
+    flexmock(module.sys).should_receive('exit').with_args(
+        module.EXIT_CODE_FROM_SIGNAL + signal_number
+    ).once()
+
+    module.handle_signal(signal_number, frame)
+
+
+def test_configure_signals_installs_signal_handlers():
+    flexmock(module.signal).should_receive('signal').at_least().once()
+
+    module.configure_signals()