Fix "borgmatic list --successful" with a slightly better heuristic for listing successful (non-checkpoint) archives.

https://yaml.org/spec/1.2/spec.html#id2780069

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.tox

.drone.yml

@@ -1,30 +1,30 @@
 ---
 kind: pipeline
-name: python-3-5-alpine-3-9
+name: python-3-5-alpine-3-10
 
 steps:
 - name: build
-  image: python:3.5-alpine3.9
+  image: python:3.5-alpine3.10
   pull: always
   commands:
     - scripts/run-tests
 ---
 kind: pipeline
-name: python-3-6-alpine-3-9
+name: python-3-6-alpine-3-10
 
 steps:
 - name: build
-  image: python:3.6-alpine3.9
+  image: python:3.6-alpine3.10
   pull: always
   commands:
     - scripts/run-tests
 ---
 kind: pipeline
-name: python-3-7-alpine-3-9
+name: python-3-7-alpine-3-10
 
 steps:
 - name: build
-  image: python:3.7-alpine3.9
+  image: python:3.7-alpine3.10
   pull: always
   commands:
     - scripts/run-tests

.eleventy.js

@@ -16,7 +16,12 @@ module.exports = function(eleventyConfig) {
         // Replace links to .md files with links to directories. This allows unparsed Markdown links
         // to work on GitHub, while rendered links elsewhere also work.
         replaceLink: function (link, env) {
-            return link.replace(/\.md$/, '/');
+            link = link.replace(/\.md$/, '/');
+
+            if (process.env.NODE_ENV == "production") {
+                return link;
+            }
+            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
         }
     };
     let markdownItAnchorOptions = {

.gitea/issue_template.md

@@ -0,0 +1,31 @@
+#### What I'm trying to do and why
+
+#### Steps to reproduce (if a bug)
+
+Include (sanitized) borgmatic configuration files if applicable.
+
+#### Actual behavior (if a bug)
+
+Include (sanitized) `--verbosity 2` output if applicable.
+
+#### Expected behavior (if a bug)
+
+#### Other notes / implementation ideas
+
+#### Environment
+
+**borgmatic version:** [version here]
+
+Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+
+**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
+
+**Borg version:** [version here]
+
+Use `sudo borg --version`
+
+**Python version:** [version here]
+
+Use `python3 --version`
+
+**operating system and version:** [OS here]

MANIFEST.in

@@ -1 +1,2 @@
 include borgmatic/config/schema.yaml
+graft sample/systemd

NEWS

@@ -1,3 +1,99 @@
+1.3.26
+ * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
+   (non-checkpoint) archives.
+
+1.3.25
+ * #223: Dead man's switch to detect when backups start failing silently, implemented via
+   healthchecks.io hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+ * Documentation on monitoring and alerting options for borgmatic backups:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
+ * Automatically rewrite links when developing on documentation locally.
+
+1.3.24
+ * #86: Add "borgmatic list --successful" flag to only list successful (non-checkpoint) archives.
+ * Add a suggestion form to all documentation pages, so users can submit ideas for improving the
+   documentation.
+ * Update documentation link to community Arch Linux borgmatic package.
+
+1.3.23
+ * #174: More detailed error alerting via runtime context available in "on_error" hook.
+
+1.3.22
+ * #144: When backups to one of several repositories fails, keep backing up to the other
+   repositories and report errors afterwards.
+
+1.3.21
+ * #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+
+1.3.20
+ * #205: More robust sample systemd service: boot delay, network dependency, lowered CPU/IO
+   priority, etc.
+ * #221: Fix "borgmatic create --progress" output so that it updates on the console in real-time.
+
+1.3.19
+ * #219: Fix visibility of "borgmatic prune --stats" output.
+
+1.3.18
+ * #220: Fix regression of argument parsing for default actions.
+
+1.3.17
+ * #217: Fix error with "borgmatic check --only" command-line flag with "extract" consistency check.
+
+1.3.16
+ * #210: Support for Borg check --verify-data flag via borgmatic "data" consistency check.
+ * #210: Override configured consistency checks via "borgmatic check --only" command-line flag.
+ * When generating sample configuration with generate-borgmatic-config, add a space after each "#"
+   comment indicator.
+
+1.3.15
+ * #208: Fix for traceback when the "checks" option has an empty value.
+ * #209: Bypass Borg error about a moved repository via "relocated_repo_access_is_ok" option in
+   borgmatic storage configuration section.
+ * #213: Reorder arguments passed to Borg to fix duplicate directories when using Borg patterns.
+ * #214: Fix for hook erroring with exit code 1 not being interpreted as an error.
+
+1.3.14
+ * #204: Do not treat Borg warnings (exit code 1) as failures.
+ * When validating configuration files, require strings instead of allowing any scalar type.
+
+1.3.13
+ * #199: Add note to documentation about using spaces instead of tabs for indentation, as YAML does
+   not allow tabs.
+ * #203: Fix compatibility with ruamel.yaml 0.16.x.
+ * If a "prefix" option in borgmatic's configuration has an empty value (blank or ""), then disable
+   default prefix.
+
+1.3.12
+ * Only log to syslog when run from a non-interactive console (e.g. a cron job).
+ * Remove unicode byte order mark from syslog output so it doesn't show up as a literal in rsyslog
+   output. See discussion on #197.
+
+1.3.11
+ * #193: Pass through several "borg list" and "borg info" flags like --short, --format, --sort-by,
+   --first, --last, etc. via borgmatic command-line flags.
+ * Add borgmatic info --repository and --archive command-line flags to display info for individual
+   repositories or archives.
+ * Support for Borg --noatime, --noctime, and --nobirthtime flags via corresponding options in
+   borgmatic configuration location section.
+
+1.3.10
+ * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.
+
+1.3.9
+ * #195: Switch to command-line actions as more traditional sub-commands, e.g. "borgmatic create",
+   "borgmatic prune", etc. However, the classic dashed options like "--create" still work!
+
+1.3.8
+ * #191: Disable console color via "color" option in borgmatic configuration output section.
+
+1.3.7
+ * #196: Fix for unclear error message for invalid YAML merge include.
+ * #197: Don't color syslog output.
+ * Change default syslog verbosity to show errors only.
+
 1.3.6
  * #53: Log to syslog in addition to existing console logging. Add --syslog-verbosity flag to
    customize the log level. See the documentation for more information:

README.md

@@ -2,7 +2,7 @@
 title: borgmatic
 permalink: index.html
 ---
-<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg)</a>
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
 
 ## Overview
 
@@ -63,8 +63,9 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
  * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
  * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
  * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
  * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
- * [Run preparation steps before backups](https://torsion.org/borgmatic/docs/how-to/run-preparation-steps-before-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
  * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
  * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 
@@ -116,8 +117,3 @@ your thing. In general, contributions are very welcome. We don't bite!
 Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
-
-<script>
-  var links = document.getElementsByClassName("referral");
-  links[Math.floor(Math.random() * links.length)].style.display = "none";
-</script>

borgmatic/borg/check.py

@@ -2,18 +2,18 @@ import logging
 
 from borgmatic.borg import extract
 from borgmatic.execute import execute_command
-from borgmatic.logger import get_logger
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
 
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
-def _parse_checks(consistency_config):
+def _parse_checks(consistency_config, only_checks=None):
     '''
-    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
+    Given a consistency config with a "checks" list, and an optional list of override checks,
+    transform them a tuple of named checks to run.
 
     For example, given a retention config of:
 
@@ -23,16 +23,21 @@ def _parse_checks(consistency_config):
 
         ('repository', 'archives')
 
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If the checks value
+    is the string "disabled", return an empty tuple, meaning that no checks should be run.
+
+    If the "data" option is present, then make sure the "archives" option is included as well.
     '''
-    checks = consistency_config.get('checks', [])
+    checks = [
+        check.lower() for check in (only_checks or consistency_config.get('checks', []) or [])
+    ]
     if checks == ['disabled']:
         return ()
 
-    return (
-        tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
-    )
+    if 'data' in checks and 'archives' not in checks:
+        checks.append('archives')
+
+    return tuple(check for check in checks if check not in ('disabled', '')) or DEFAULT_CHECKS
 
 
 def _make_check_flags(checks, check_last=None, prefix=None):
@@ -56,7 +61,7 @@ def _make_check_flags(checks, check_last=None, prefix=None):
     '''
     if 'archives' in checks:
         last_flags = ('--last', str(check_last)) if check_last else ()
-        prefix_flags = ('--prefix', prefix) if prefix else ('--prefix', DEFAULT_PREFIX)
+        prefix_flags = ('--prefix', prefix) if prefix else ()
     else:
         last_flags = ()
         prefix_flags = ()
@@ -69,30 +74,37 @@ def _make_check_flags(checks, check_last=None, prefix=None):
                 'Ignoring consistency prefix option, as "archives" is not in consistency checks.'
             )
 
+    common_flags = last_flags + prefix_flags + (('--verify-data',) if 'data' in checks else ())
+
     if set(DEFAULT_CHECKS).issubset(set(checks)):
-        return last_flags + prefix_flags
+        return common_flags
 
     return (
         tuple('--{}-only'.format(check) for check in checks if check in DEFAULT_CHECKS)
-        + last_flags
-        + prefix_flags
+        + common_flags
     )
 
 
 def check_archives(
-    repository, storage_config, consistency_config, local_path='borg', remote_path=None
+    repository,
+    storage_config,
+    consistency_config,
+    local_path='borg',
+    remote_path=None,
+    only_checks=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, a consistency config dict,
-    and a local/remote commands to run, check the contained Borg archives for consistency.
+    local/remote commands to run, and an optional list of checks to use instead of configured
+    checks, check the contained Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
-    checks = _parse_checks(consistency_config)
+    checks = _parse_checks(consistency_config, only_checks)
     check_last = consistency_config.get('check_last', None)
     lock_wait = None
 
-    if set(checks).intersection(set(DEFAULT_CHECKS)):
+    if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
         remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
         lock_wait = storage_config.get('lock_wait', None)
         lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
@@ -103,14 +115,15 @@ def check_archives(
         if logger.isEnabledFor(logging.DEBUG):
             verbosity_flags = ('--debug', '--show-rc')
 
-        prefix = consistency_config.get('prefix')
+        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
 
         full_command = (
-            (local_path, 'check', repository)
+            (local_path, 'check')
             + _make_check_flags(checks, check_last, prefix)
             + remote_path_flags
             + lock_wait_flags
             + verbosity_flags
+            + (repository,)
         )
 
         execute_command(full_command)

borgmatic/borg/create.py

@@ -4,10 +4,9 @@ import logging
 import os
 import tempfile
 
-from borgmatic.execute import execute_command
-from borgmatic.logger import get_logger
+from borgmatic.execute import execute_command, execute_command_without_capture
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _expand_directory(directory):
@@ -127,14 +126,7 @@ def create_archive(
     archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
 
     full_command = (
-        (
-            local_path,
-            'create',
-            '{repository}::{archive_name_format}'.format(
-                repository=repository, archive_name_format=archive_name_format
-            ),
-        )
-        + sources
+        (local_path, 'create')
         + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
         + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
@@ -143,6 +135,9 @@ def create_archive(
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--noatime',) if location_config.get('atime') is False else ())
+        + (('--noctime',) if location_config.get('ctime') is False else ())
+        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
         + (('--read-special',) if location_config.get('read_special') else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
@@ -160,8 +155,20 @@ def create_archive(
         + (('--dry-run',) if dry_run else ())
         + (('--progress',) if progress else ())
         + (('--json',) if json else ())
+        + (
+            '{repository}::{archive_name_format}'.format(
+                repository=repository, archive_name_format=archive_name_format
+            ),
+        )
+        + sources
     )
 
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(full_command)
+        return
+
     if json:
         output_log_level = None
     elif stats:

borgmatic/borg/environment.py

@@ -11,9 +11,21 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'ssh_command': 'BORG_RSH',
 }
 
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
+}
+
 
 def initialize(storage_config):
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
         value = storage_config.get(option_name)
         if value:
             os.environ[environment_variable_name] = value
+
+    for (
+        option_name,
+        environment_variable_name,
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name, False)
+        os.environ[environment_variable_name] = 'yes' if value else 'no'

borgmatic/borg/extract.py

@@ -1,9 +1,8 @@
 import logging
 
-from borgmatic.execute import execute_command
-from borgmatic.logger import get_logger
+from borgmatic.execute import execute_command, execute_command_without_capture
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg', remote_path=None):
@@ -20,10 +19,11 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         verbosity_flags = ('--info',)
 
     full_list_command = (
-        (local_path, 'list', '--short', repository)
+        (local_path, 'list', '--short')
         + remote_path_flags
         + lock_wait_flags
         + verbosity_flags
+        + (repository,)
     )
 
     list_output = execute_command(full_list_command, output_log_level=None)
@@ -35,18 +35,16 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
     full_extract_command = (
-        (
-            local_path,
-            'extract',
-            '--dry-run',
-            '{repository}::{last_archive_name}'.format(
-                repository=repository, last_archive_name=last_archive_name
-            ),
-        )
+        (local_path, 'extract', '--dry-run')
         + remote_path_flags
         + lock_wait_flags
         + verbosity_flags
         + list_flag
+        + (
+            '{repository}::{last_archive_name}'.format(
+                repository=repository, last_archive_name=last_archive_name
+            ),
+        )
     )
 
     execute_command(full_extract_command)
@@ -72,8 +70,7 @@ def extract_archive(
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'extract', '::'.join((repository, archive)))
-        + (tuple(restore_paths) if restore_paths else ())
+        (local_path, 'extract')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--umask', str(umask)) if umask else ())
@@ -82,6 +79,14 @@ def extract_archive(
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--progress',) if progress else ())
+        + ('::'.join((repository, archive)),)
+        + (tuple(restore_paths) if restore_paths else ())
     )
 
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(full_command)
+        return
+
     execute_command(full_command)

borgmatic/borg/flags.py

@@ -0,0 +1,31 @@
+import itertools
+
+
+def make_flags(name, value):
+    '''
+    Given a flag name and its value, return it formatted as Borg-compatible flags.
+    '''
+    if not value:
+        return ()
+
+    flag = '--{}'.format(name.replace('_', '-'))
+
+    if value is True:
+        return (flag,)
+
+    return (flag, str(value))
+
+
+def make_flags_from_arguments(arguments, excludes=()):
+    '''
+    Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
+    list of named arguments to exclude, generate and return the corresponding Borg command-line
+    flags as a tuple.
+    '''
+    return tuple(
+        itertools.chain.from_iterable(
+            make_flags(name, value=getattr(arguments, name))
+            for name in sorted(vars(arguments))
+            if name not in excludes and not name.startswith('_')
+        )
+    )

borgmatic/borg/info.py

@@ -1,27 +1,43 @@
 import logging
 
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
-from borgmatic.logger import get_logger
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def display_archives_info(
-    repository, storage_config, local_path='borg', remote_path=None, json=False
+    repository, storage_config, info_arguments, local_path='borg', remote_path=None
 ):
     '''
-    Given a local or remote repository path, and a storage config dict, display summary information
-    for Borg archives in the repository or return JSON summary information.
+    Given a local or remote repository path, a storage config dict, and the arguments to the info
+    action, display summary information for Borg archives in the repository or return JSON summary
+    information.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'info', repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
-        + (('--json',) if json else ())
+        (local_path, 'info')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not info_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not info_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
+        + (
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
+        )
     )
 
-    return execute_command(full_command, output_log_level=None if json else logging.WARNING)
+    return execute_command(
+        full_command, output_log_level=None if info_arguments.json else logging.WARNING
+    )

borgmatic/borg/init.py

@@ -1,10 +1,9 @@
 import logging
 import subprocess
 
-from borgmatic.execute import execute_command
-from borgmatic.logger import get_logger
+from borgmatic.execute import execute_command, execute_command_without_capture
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
@@ -35,14 +34,15 @@ def initialize_repository(
             raise
 
     init_command = (
-        (local_path, 'init', repository)
+        (local_path, 'init')
         + (('--encryption', encryption_mode) if encryption_mode else ())
         + (('--append-only',) if append_only else ())
         + (('--storage-quota', storage_quota) if storage_quota else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--remote-path', remote_path) if remote_path else ())
+        + (repository,)
     )
 
     # Don't use execute_command() here because it doesn't support interactive prompts.
-    subprocess.check_call(init_command)
+    execute_command_without_capture(init_command)

borgmatic/borg/list.py

@@ -1,28 +1,50 @@
 import logging
 
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
-from borgmatic.logger import get_logger
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
-def list_archives(
-    repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
-):
+# A hack to convince Borg to exclude archives ending in ".checkpoint". This assumes that a
+# non-checkpoint archive name ends in a digit (e.g. from a timestamp).
+BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
+
+
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
-    Given a local or remote repository path and a storage config dict, display the output of listing
-    Borg archives in the repository or return JSON output. Or, if an archive name is given, listing
-    the files in that archive.
+    Given a local or remote repository path, a storage config dict, and the arguments to the list
+    action, display the output of listing Borg archives in the repository or return JSON output. Or,
+    if an archive name is given, listing the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
+    if list_arguments.successful:
+        list_arguments.glob_archives = BORG_EXCLUDE_CHECKPOINTS_GLOB
 
     full_command = (
-        (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
-        + (('--json',) if json else ())
+        (local_path, 'list')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(
+            list_arguments, excludes=('repository', 'archive', 'successful')
+        )
+        + (
+            '::'.join((repository, list_arguments.archive))
+            if list_arguments.archive
+            else repository,
+        )
     )
 
-    return execute_command(full_command, output_log_level=None if json else logging.WARNING)
+    return execute_command(
+        full_command, output_log_level=None if list_arguments.json else logging.WARNING
+    )

borgmatic/borg/prune.py

@@ -1,9 +1,8 @@
 import logging
 
 from borgmatic.execute import execute_command
-from borgmatic.logger import get_logger
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _make_prune_flags(retention_config):
@@ -22,12 +21,15 @@ def _make_prune_flags(retention_config):
             ('--keep-monthly', '6'),
         )
     '''
-    if not retention_config.get('prefix'):
-        retention_config['prefix'] = '{hostname}-'
+    config = retention_config.copy()
+
+    if 'prefix' not in config:
+        config['prefix'] = '{hostname}-'
+    elif not config['prefix']:
+        config.pop('prefix')
 
     return (
-        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
-        for option_name, value in retention_config.items()
+        ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
     )
 
 
@@ -49,7 +51,7 @@ def prune_archives(
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'prune', repository)
+        (local_path, 'prune')
         + tuple(element for pair in _make_prune_flags(retention_config) for element in pair)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
@@ -59,6 +61,7 @@ def prune_archives(
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--stats',) if stats else ())
+        + (repository,)
     )
 
-    execute_command(full_command)
+    execute_command(full_command, output_log_level=logging.WARNING if stats else logging.INFO)

borgmatic/commands/arguments.py

@@ -0,0 +1,408 @@
+import collections
+from argparse import ArgumentParser
+
+from borgmatic.config import collect
+
+SUBPARSER_ALIASES = {
+    'init': ['--init', '-I'],
+    'prune': ['--prune', '-p'],
+    'create': ['--create', '-C'],
+    'check': ['--check', '-k'],
+    'extract': ['--extract', '-x'],
+    'list': ['--list', '-l'],
+    'info': ['--info', '-i'],
+}
+
+
+def parse_subparser_arguments(unparsed_arguments, subparsers):
+    '''
+    Given a sequence of arguments, and a subparsers object as returned by
+    argparse.ArgumentParser().add_subparsers(), give each requested action's subparser a shot at
+    parsing all arguments. This allows common arguments like "--repository" to be shared across
+    multiple subparsers.
+
+    Return the result as a dict mapping from subparser name to a parsed namespace of arguments.
+    '''
+    arguments = collections.OrderedDict()
+    remaining_arguments = list(unparsed_arguments)
+    alias_to_subparser_name = {
+        alias: subparser_name
+        for subparser_name, aliases in SUBPARSER_ALIASES.items()
+        for alias in aliases
+    }
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name)
+
+        # If a parsed value happens to be the same as the name of a subparser, remove it from the
+        # remaining arguments. This prevents, for instance, "check --only extract" from triggering
+        # the "extract" subparser.
+        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+        for value in vars(parsed).values():
+            if isinstance(value, str):
+                if value in subparsers.choices:
+                    remaining_arguments.remove(value)
+            elif isinstance(value, list):
+                for item in value:
+                    if item in subparsers.choices:
+                        remaining_arguments.remove(item)
+
+        arguments[canonical_name] = parsed
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+            arguments[subparser_name] = parsed
+
+    return arguments
+
+
+def parse_global_arguments(unparsed_arguments, top_level_parser, subparsers):
+    '''
+    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
+    object as returned by argparse.ArgumentParser().add_subparsers(), parse and return any global
+    arguments as a parsed argparse.Namespace instance.
+    '''
+    # Ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
+    # are global arguments.
+    remaining_arguments = list(unparsed_arguments)
+    present_subparser_names = set()
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        present_subparser_names.add(subparser_name)
+        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if (
+        not present_subparser_names
+        and '--help' not in unparsed_arguments
+        and '-h' not in unparsed_arguments
+    ):
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # Remove the subparser names themselves.
+    for subparser_name in present_subparser_names:
+        if subparser_name in remaining_arguments:
+            remaining_arguments.remove(subparser_name)
+
+    return top_level_parser.parse_args(remaining_arguments)
+
+
+def parse_arguments(*unparsed_arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    global_parser = ArgumentParser(add_help=False)
+    global_group = global_parser.add_argument_group('global arguments')
+
+    global_group.add_argument(
+        '-c',
+        '--config',
+        nargs='*',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+    global_group.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    global_group.add_argument(
+        '-n',
+        '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write to any repositories',
+    )
+    global_group.add_argument(
+        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
+    )
+    global_group.add_argument(
+        '-v',
+        '--verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to the console (from none to lots: 0, 1, or 2)',
+    )
+    global_group.add_argument(
+        '--syslog-verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive',
+    )
+    global_group.add_argument(
+        '--version',
+        dest='version',
+        default=False,
+        action='store_true',
+        help='Display installed version number of borgmatic and exit',
+    )
+
+    top_level_parser = ArgumentParser(
+        description='''
+            A simple wrapper script for the Borg backup software that creates and prunes backups.
+            If none of the action options are given, then borgmatic defaults to: prune, create, and
+            check archives.
+            ''',
+        parents=[global_parser],
+    )
+
+    subparsers = top_level_parser.add_subparsers(
+        title='actions',
+        metavar='',
+        help='Specify zero or more actions. Defaults to prune, create, and check. Use --help with action for details:',
+    )
+    init_parser = subparsers.add_parser(
+        'init',
+        aliases=SUBPARSER_ALIASES['init'],
+        help='Initialize an empty Borg repository',
+        description='Initialize an empty Borg repository',
+        add_help=False,
+    )
+    init_group = init_parser.add_argument_group('init arguments')
+    init_group.add_argument(
+        '-e',
+        '--encryption',
+        dest='encryption_mode',
+        help='Borg repository encryption mode',
+        required=True,
+    )
+    init_group.add_argument(
+        '--append-only',
+        dest='append_only',
+        action='store_true',
+        help='Create an append-only repository',
+    )
+    init_group.add_argument(
+        '--storage-quota',
+        dest='storage_quota',
+        help='Create a repository with a fixed storage quota',
+    )
+    init_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    prune_parser = subparsers.add_parser(
+        'prune',
+        aliases=SUBPARSER_ALIASES['prune'],
+        help='Prune archives according to the retention policy',
+        description='Prune archives according to the retention policy',
+        add_help=False,
+    )
+    prune_group = prune_parser.add_argument_group('prune arguments')
+    prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    create_parser = subparsers.add_parser(
+        'create',
+        aliases=SUBPARSER_ALIASES['create'],
+        help='Create archives (actually perform backups)',
+        description='Create archives (actually perform backups)',
+        add_help=False,
+    )
+    create_group = create_parser.add_argument_group('create arguments')
+    create_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    create_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    check_parser = subparsers.add_parser(
+        'check',
+        aliases=SUBPARSER_ALIASES['check'],
+        help='Check archives for consistency',
+        description='Check archives for consistency',
+        add_help=False,
+    )
+    check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--only',
+        metavar='CHECK',
+        choices=('repository', 'archives', 'data', 'extract'),
+        dest='only',
+        action='append',
+        help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks; can specify flag multiple times',
+    )
+    check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    extract_parser = subparsers.add_parser(
+        'extract',
+        aliases=SUBPARSER_ALIASES['extract'],
+        help='Extract a named archive to the current directory',
+        description='Extract a named archive to the current directory',
+        add_help=False,
+    )
+    extract_group = extract_parser.add_argument_group('extract arguments')
+    extract_group.add_argument(
+        '--repository',
+        help='Path of repository to extract, defaults to the configured repository if there is only one',
+    )
+    extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    extract_group.add_argument(
+        '--restore-path',
+        nargs='+',
+        dest='restore_paths',
+        help='Paths to restore from archive, defaults to the entire archive',
+    )
+    extract_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    extract_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    list_parser = subparsers.add_parser(
+        'list',
+        aliases=SUBPARSER_ALIASES['list'],
+        help='List archives',
+        description='List archives or the contents of an archive',
+        add_help=False,
+    )
+    list_group = list_parser.add_argument_group('list arguments')
+    list_group.add_argument(
+        '--repository',
+        help='Path of repository to list, defaults to the configured repository if there is only one',
+    )
+    list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument(
+        '--short', default=False, action='store_true', help='Output only archive or path names'
+    )
+    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument(
+        '--json', default=False, action='store_true', help='Output results as JSON'
+    )
+    list_group.add_argument(
+        '-P', '--prefix', help='Only list archive names starting with this prefix'
+    )
+    list_group.add_argument(
+        '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
+    )
+    list_group.add_argument(
+        '--successful',
+        default=False,
+        action='store_true',
+        help='Only list archive names of successful (non-checkpoint) backups',
+    )
+    list_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    list_group.add_argument(
+        '--first', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '--last', metavar='N', help='List last N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+    )
+    list_group.add_argument(
+        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+    )
+    list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
+    list_group.add_argument(
+        '--pattern-from',
+        metavar='FILENAME',
+        help='Include or exclude paths matching patterns from pattern file, one per line',
+    )
+    list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    info_parser = subparsers.add_parser(
+        'info',
+        aliases=SUBPARSER_ALIASES['info'],
+        help='Display summary information on archives',
+        description='Display summary information on archives',
+        add_help=False,
+    )
+    info_group = info_parser.add_argument_group('info arguments')
+    info_group.add_argument(
+        '--repository',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one',
+    )
+    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    info_group.add_argument(
+        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+    )
+    info_group.add_argument(
+        '-a',
+        '--glob-archives',
+        metavar='GLOB',
+        help='Only show info for archive names matching this glob',
+    )
+    info_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    info_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Show info for first N archives after other filters are applied',
+    )
+    info_group.add_argument(
+        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+    )
+    info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments = parse_subparser_arguments(unparsed_arguments, subparsers)
+    arguments['global'] = parse_global_arguments(unparsed_arguments, top_level_parser, subparsers)
+
+    if arguments['global'].excludes_filename:
+        raise ValueError(
+            'The --excludes option has been replaced with exclude_patterns in configuration'
+        )
+
+    if 'init' in arguments and arguments['global'].dry_run:
+        raise ValueError('The init action cannot be used with the --dry-run option')
+
+    if 'list' in arguments and arguments['list'].glob_archives and arguments['list'].successful:
+        raise ValueError('The --glob-archives and --successful options cannot be used together')
+
+    if (
+        'list' in arguments
+        and 'info' in arguments
+        and arguments['list'].json
+        and arguments['info'].json
+    ):
+        raise ValueError('With the --json option, list and info actions cannot be used together')
+
+    return arguments

borgmatic/commands/borgmatic.py

@@ -3,7 +3,6 @@ import json
 import logging
 import os
 import sys
-from argparse import ArgumentParser
 from subprocess import CalledProcessError
 
 import colorama
@@ -18,305 +17,127 @@ from borgmatic.borg import info as borg_info
 from borgmatic.borg import init as borg_init
 from borgmatic.borg import list as borg_list
 from borgmatic.borg import prune as borg_prune
+from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
-from borgmatic.logger import configure_logging, get_logger, should_do_markup
+from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
 
 
-def parse_arguments(*arguments):
+def run_configuration(config_filename, config, arguments):
     '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an argparse.ArgumentParser instance.
-    '''
-    config_paths = collect.get_default_config_paths()
-
-    parser = ArgumentParser(
-        description='''
-            A simple wrapper script for the Borg backup software that creates and prunes backups.
-            If none of the action options are given, then borgmatic defaults to: prune, create, and
-            check archives.
-            ''',
-        add_help=False,
-    )
-
-    actions_group = parser.add_argument_group('actions')
-    actions_group.add_argument(
-        '-I', '--init', dest='init', action='store_true', help='Initialize an empty Borg repository'
-    )
-    actions_group.add_argument(
-        '-p',
-        '--prune',
-        dest='prune',
-        action='store_true',
-        help='Prune archives according to the retention policy',
-    )
-    actions_group.add_argument(
-        '-C',
-        '--create',
-        dest='create',
-        action='store_true',
-        help='Create archives (actually perform backups)',
-    )
-    actions_group.add_argument(
-        '-k', '--check', dest='check', action='store_true', help='Check archives for consistency'
-    )
-
-    actions_group.add_argument(
-        '-x',
-        '--extract',
-        dest='extract',
-        action='store_true',
-        help='Extract a named archive to the current directory',
-    )
-    actions_group.add_argument(
-        '-l', '--list', dest='list', action='store_true', help='List archives'
-    )
-    actions_group.add_argument(
-        '-i',
-        '--info',
-        dest='info',
-        action='store_true',
-        help='Display summary information on archives',
-    )
-
-    init_group = parser.add_argument_group('options for --init')
-    init_group.add_argument(
-        '-e', '--encryption', dest='encryption_mode', help='Borg repository encryption mode'
-    )
-    init_group.add_argument(
-        '--append-only',
-        dest='append_only',
-        action='store_true',
-        help='Create an append-only repository',
-    )
-    init_group.add_argument(
-        '--storage-quota',
-        dest='storage_quota',
-        help='Create a repository with a fixed storage quota',
-    )
-
-    prune_group = parser.add_argument_group('options for --prune')
-    stats_argument = prune_group.add_argument(
-        '--stats',
-        dest='stats',
-        default=False,
-        action='store_true',
-        help='Display statistics of archive',
-    )
-
-    create_group = parser.add_argument_group('options for --create')
-    progress_argument = create_group.add_argument(
-        '--progress',
-        dest='progress',
-        default=False,
-        action='store_true',
-        help='Display progress for each file as it is processed',
-    )
-    create_group._group_actions.append(stats_argument)
-    json_argument = create_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
-    )
-
-    extract_group = parser.add_argument_group('options for --extract')
-    repository_argument = extract_group.add_argument(
-        '--repository',
-        help='Path of repository to use, defaults to the configured repository if there is only one',
-    )
-    archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
-    extract_group.add_argument(
-        '--restore-path',
-        nargs='+',
-        dest='restore_paths',
-        help='Paths to restore from archive, defaults to the entire archive',
-    )
-    extract_group._group_actions.append(progress_argument)
-
-    list_group = parser.add_argument_group('options for --list')
-    list_group._group_actions.append(repository_argument)
-    list_group._group_actions.append(archive_argument)
-    list_group._group_actions.append(json_argument)
-
-    info_group = parser.add_argument_group('options for --info')
-    info_group._group_actions.append(json_argument)
-
-    common_group = parser.add_argument_group('common options')
-    common_group.add_argument(
-        '-c',
-        '--config',
-        nargs='+',
-        dest='config_paths',
-        default=config_paths,
-        help='Configuration filenames or directories, defaults to: {}'.format(
-            ' '.join(config_paths)
-        ),
-    )
-    common_group.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        help='Deprecated in favor of exclude_patterns within configuration',
-    )
-    common_group.add_argument(
-        '-n',
-        '--dry-run',
-        dest='dry_run',
-        action='store_true',
-        help='Go through the motions, but do not actually write to any repositories',
-    )
-    common_group.add_argument(
-        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
-    )
-    common_group.add_argument(
-        '-v',
-        '--verbosity',
-        type=int,
-        choices=range(0, 3),
-        default=0,
-        help='Display verbose progress to the console (from none to lots: 0, 1, or 2)',
-    )
-    common_group.add_argument(
-        '--syslog-verbosity',
-        type=int,
-        choices=range(0, 3),
-        default=1,
-        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2)',
-    )
-    common_group.add_argument(
-        '--version',
-        dest='version',
-        default=False,
-        action='store_true',
-        help='Display installed version number of borgmatic and exit',
-    )
-    common_group.add_argument('--help', action='help', help='Show this help information and exit')
-
-    args = parser.parse_args(arguments)
-
-    if args.excludes_filename:
-        raise ValueError(
-            'The --excludes option has been replaced with exclude_patterns in configuration'
-        )
-
-    if (args.encryption_mode or args.append_only or args.storage_quota) and not args.init:
-        raise ValueError(
-            'The --encryption, --append-only, and --storage-quota options can only be used with the --init option'
-        )
-
-    if args.init and args.dry_run:
-        raise ValueError('The --init option cannot be used with the --dry-run option')
-    if args.init and not args.encryption_mode:
-        raise ValueError('The --encryption option is required with the --init option')
-
-    if not args.extract:
-        if not args.list:
-            if args.repository:
-                raise ValueError(
-                    'The --repository option can only be used with the --extract and --list options'
-                )
-            if args.archive:
-                raise ValueError(
-                    'The --archive option can only be used with the --extract and --list options'
-                )
-        if args.restore_paths:
-            raise ValueError('The --restore-path option can only be used with the --extract option')
-    if args.extract and not args.archive:
-        raise ValueError('The --archive option is required with the --extract option')
-
-    if args.progress and not (args.create or args.extract):
-        raise ValueError(
-            'The --progress option can only be used with the --create and --extract options'
-        )
-
-    if args.json and not (args.create or args.list or args.info):
-        raise ValueError(
-            'The --json option can only be used with the --create, --list, or --info options'
-        )
-
-    if args.json and args.list and args.info:
-        raise ValueError(
-            'With the --json option, options --list and --info cannot be used together'
-        )
-
-    # If any of the action flags are explicitly requested, leave them as-is. Otherwise, assume
-    # defaults: Mutate the given arguments to enable the default actions.
-    if (
-        not args.init
-        and not args.prune
-        and not args.create
-        and not args.check
-        and not args.extract
-        and not args.list
-        and not args.info
-    ):
-        args.prune = True
-        args.create = True
-        args.check = True
-
-    if args.stats and not (args.create or args.prune):
-        raise ValueError('The --stats option can only be used when creating or pruning archives')
-
-    return args
-
-
-def run_configuration(config_filename, config, args):  # pragma: no cover
-    '''
-    Given a config filename and the corresponding parsed config dict, execute its defined pruning,
+    Given a config filename, the corresponding parsed config dict, and command-line arguments as a
+    dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
     backups, consistency checks, and/or other actions.
 
-    Yield JSON output strings from executing any actions that produce JSON.
+    Yield a combination of:
+
+      * JSON output strings from successfully executing any actions that produce JSON
+      * logging.LogRecord instances containing errors from any actions or backup hooks that fail
     '''
     (location, storage, retention, consistency, hooks) = (
         config.get(section_name, {})
         for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
     )
+    global_arguments = arguments['global']
 
-    try:
-        local_path = location.get('local_path', 'borg')
-        remote_path = location.get('remote_path')
-        borg_environment.initialize(storage)
+    local_path = location.get('local_path', 'borg')
+    remote_path = location.get('remote_path')
+    borg_environment.initialize(storage)
+    encountered_error = None
+    error_repository = ''
 
-        if args.create:
+    if 'create' in arguments:
+        try:
             hook.execute_hook(
                 hooks.get('before_backup'),
                 hooks.get('umask'),
                 config_filename,
                 'pre-backup',
-                args.dry_run,
+                global_arguments.dry_run,
+            )
+            hook.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
+            )
+        except (OSError, CalledProcessError) as error:
+            encountered_error = error
+            yield from make_error_log_records(
+                '{}: Error running pre-backup hook'.format(config_filename), error
             )
 
+    if not encountered_error:
         for repository_path in location['repositories']:
-            yield from run_actions(
-                args=args,
-                location=location,
-                storage=storage,
-                retention=retention,
-                consistency=consistency,
-                local_path=local_path,
-                remote_path=remote_path,
-                repository_path=repository_path,
-            )
+            try:
+                yield from run_actions(
+                    arguments=arguments,
+                    location=location,
+                    storage=storage,
+                    retention=retention,
+                    consistency=consistency,
+                    local_path=local_path,
+                    remote_path=remote_path,
+                    repository_path=repository_path,
+                )
+            except (OSError, CalledProcessError) as error:
+                encountered_error = error
+                error_repository = repository_path
+                yield from make_error_log_records(
+                    '{}: Error running actions for repository'.format(repository_path), error
+                )
 
-        if args.create:
+    if 'create' in arguments and not encountered_error:
+        try:
             hook.execute_hook(
                 hooks.get('after_backup'),
                 hooks.get('umask'),
                 config_filename,
                 'post-backup',
-                args.dry_run,
+                global_arguments.dry_run,
+            )
+            hook.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run
+            )
+        except (OSError, CalledProcessError) as error:
+            encountered_error = error
+            yield from make_error_log_records(
+                '{}: Error running post-backup hook'.format(config_filename), error
+            )
+
+    if encountered_error:
+        try:
+            hook.execute_hook(
+                hooks.get('on_error'),
+                hooks.get('umask'),
+                config_filename,
+                'on-error',
+                global_arguments.dry_run,
+                repository=error_repository,
+                error=encountered_error,
+                output=getattr(encountered_error, 'output', ''),
+            )
+            hook.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
+            )
+        except (OSError, CalledProcessError) as error:
+            yield from make_error_log_records(
+                '{}: Error running on-error hook'.format(config_filename), error
             )
-    except (OSError, CalledProcessError):
-        hook.execute_hook(
-            hooks.get('on_error'), hooks.get('umask'), config_filename, 'on-error', args.dry_run
-        )
-        raise
 
 
 def run_actions(
-    *, args, location, storage, retention, consistency, local_path, remote_path, repository_path
+    *,
+    arguments,
+    location,
+    storage,
+    retention,
+    consistency,
+    local_path,
+    remote_path,
+    repository_path
 ):  # pragma: no cover
     '''
     Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
@@ -326,130 +147,221 @@ def run_actions(
     Yield JSON output strings from executing any actions that produce JSON.
     '''
     repository = os.path.expanduser(repository_path)
-    dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
-    if args.init:
+    global_arguments = arguments['global']
+    dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
+    if 'init' in arguments:
         logger.info('{}: Initializing repository'.format(repository))
         borg_init.initialize_repository(
             repository,
-            args.encryption_mode,
-            args.append_only,
-            args.storage_quota,
+            arguments['init'].encryption_mode,
+            arguments['init'].append_only,
+            arguments['init'].storage_quota,
             local_path=local_path,
             remote_path=remote_path,
         )
-    if args.prune:
+    if 'prune' in arguments:
         logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
         borg_prune.prune_archives(
-            args.dry_run,
+            global_arguments.dry_run,
             repository,
             storage,
             retention,
             local_path=local_path,
             remote_path=remote_path,
-            stats=args.stats,
+            stats=arguments['prune'].stats,
         )
-    if args.create:
+    if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
         json_output = borg_create.create_archive(
-            args.dry_run,
+            global_arguments.dry_run,
             repository,
             location,
             storage,
             local_path=local_path,
             remote_path=remote_path,
-            progress=args.progress,
-            stats=args.stats,
-            json=args.json,
+            progress=arguments['create'].progress,
+            stats=arguments['create'].stats,
+            json=arguments['create'].json,
         )
         if json_output:
             yield json.loads(json_output)
-    if args.check and checks.repository_enabled_for_checks(repository, consistency):
+    if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
-            repository, storage, consistency, local_path=local_path, remote_path=remote_path
+            repository,
+            storage,
+            consistency,
+            local_path=local_path,
+            remote_path=remote_path,
+            only_checks=arguments['check'].only,
         )
-    if args.extract:
-        if args.repository is None or repository == args.repository:
-            logger.info('{}: Extracting archive {}'.format(repository, args.archive))
+    if 'extract' in arguments:
+        if arguments['extract'].repository is None or repository == arguments['extract'].repository:
+            logger.info(
+                '{}: Extracting archive {}'.format(repository, arguments['extract'].archive)
+            )
             borg_extract.extract_archive(
-                args.dry_run,
+                global_arguments.dry_run,
                 repository,
-                args.archive,
-                args.restore_paths,
+                arguments['extract'].archive,
+                arguments['extract'].restore_paths,
                 location,
                 storage,
                 local_path=local_path,
                 remote_path=remote_path,
-                progress=args.progress,
+                progress=arguments['extract'].progress,
             )
-    if args.list:
-        if args.repository is None or repository == args.repository:
+    if 'list' in arguments:
+        if arguments['list'].repository is None or repository == arguments['list'].repository:
             logger.info('{}: Listing archives'.format(repository))
             json_output = borg_list.list_archives(
                 repository,
                 storage,
-                args.archive,
+                list_arguments=arguments['list'],
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            if json_output:
+                yield json.loads(json_output)
+    if 'info' in arguments:
+        if arguments['info'].repository is None or repository == arguments['info'].repository:
+            logger.info('{}: Displaying summary info for archives'.format(repository))
+            json_output = borg_info.display_archives_info(
+                repository,
+                storage,
+                info_arguments=arguments['info'],
                 local_path=local_path,
                 remote_path=remote_path,
-                json=args.json,
             )
             if json_output:
                 yield json.loads(json_output)
-    if args.info:
-        logger.info('{}: Displaying summary info for archives'.format(repository))
-        json_output = borg_info.display_archives_info(
-            repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
-        )
-        if json_output:
-            yield json.loads(json_output)
 
 
-def collect_configuration_run_summary_logs(config_filenames, args):
+def load_configurations(config_filenames):
     '''
-    Given a sequence of configuration filenames and parsed command-line arguments as an
-    argparse.ArgumentParser instance, run each configuration file and yield a series of
-    logging.LogRecord instances containing summary information about each run.
-
-    As a side effect of running through these configuration files, output their JSON results, if
-    any, to stdout.
+    Given a sequence of configuration filenames, load and validate each configuration file. Return
+    the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
+    and sequence of logging.LogRecord instances containing any parse errors.
     '''
     # Dict mapping from config filename to corresponding parsed config dict.
     configs = collections.OrderedDict()
+    logs = []
 
     # Parse and load each configuration file.
     for config_filename in config_filenames:
         try:
-            logger.info('{}: Parsing configuration file'.format(config_filename))
             configs[config_filename] = validate.parse_configuration(
                 config_filename, validate.schema_filename()
             )
         except (ValueError, OSError, validate.Validation_error) as error:
-            yield logging.makeLogRecord(
-                dict(
-                    levelno=logging.CRITICAL,
-                    levelname='CRITICAL',
-                    msg='{}: Error parsing configuration file'.format(config_filename),
-                )
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            logs.extend(
+                [
+                    logging.makeLogRecord(
+                        dict(
+                            levelno=logging.CRITICAL,
+                            levelname='CRITICAL',
+                            msg='{}: Error parsing configuration file'.format(config_filename),
+                        )
+                    ),
+                    logging.makeLogRecord(
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+                    ),
+                ]
             )
 
+    return (configs, logs)
+
+
+def make_error_log_records(message, error=None):
+    '''
+    Given error message text and an optional exception object, yield a series of logging.LogRecord
+    instances with error summary information.
+    '''
+    if not error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        return
+
+    try:
+        raise error
+    except CalledProcessError as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
+        )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except (ValueError, OSError) as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except:  # noqa: E722
+        # Raising above only as a means of determining the error type. Swallow the exception here
+        # because we don't want the exception to propagate out of this function.
+        pass
+
+
+def collect_configuration_run_summary_logs(configs, arguments):
+    '''
+    Given a dict of configuration filename to corresponding parsed configuration, and parsed
+    command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
+    each configuration file and yield a series of logging.LogRecord instances containing summary
+    information about each run.
+
+    As a side effect of running through these configuration files, output their JSON results, if
+    any, to stdout.
+    '''
     # Run cross-file validation checks.
-    if args.extract or (args.list and args.archive):
+    if 'extract' in arguments:
+        repository = arguments['extract'].repository
+    elif 'list' in arguments and arguments['list'].archive:
+        repository = arguments['list'].repository
+    else:
+        repository = None
+
+    if repository:
         try:
-            validate.guard_configuration_contains_repository(args.repository, configs)
+            validate.guard_configuration_contains_repository(repository, configs)
         except ValueError as error:
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
-            )
+            yield from make_error_log_records(str(error))
+            return
+
+    if not configs:
+        yield from make_error_log_records(
+            '{}: No configuration files found'.format(' '.join(arguments['global'].config_paths))
+        )
+        return
+
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                hook.execute_hook(
+                    hooks.get('before_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'pre-everything',
+                    arguments['global'].dry_run,
+                )
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running pre-everything hook', error)
             return
 
     # Execute the actions corresponding to each configuration file.
     json_results = []
     for config_filename, config in configs.items():
-        try:
-            json_results.extend(list(run_configuration(config_filename, config, args)))
+        results = list(run_configuration(config_filename, config, arguments))
+        error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
+
+        if error_logs:
+            yield from make_error_log_records(
+                '{}: Error running configuration file'.format(config_filename)
+            )
+            yield from error_logs
+        else:
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.INFO,
@@ -457,29 +369,25 @@ def collect_configuration_run_summary_logs(config_filenames, args):
                     msg='{}: Successfully ran configuration file'.format(config_filename),
                 )
             )
-        except (ValueError, OSError, CalledProcessError) as error:
-            yield logging.makeLogRecord(
-                dict(
-                    levelno=logging.CRITICAL,
-                    levelname='CRITICAL',
-                    msg='{}: Error running configuration file'.format(config_filename),
-                )
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
-            )
+            if results:
+                json_results.extend(results)
 
     if json_results:
         sys.stdout.write(json.dumps(json_results))
 
-    if not config_filenames:
-        yield logging.makeLogRecord(
-            dict(
-                levelno=logging.CRITICAL,
-                levelname='CRITICAL',
-                msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
-            )
-        )
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                hook.execute_hook(
+                    hooks.get('after_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-everything',
+                    arguments['global'].dry_run,
+                )
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running post-everything hook', error)
 
 
 def exit_with_help_link():  # pragma: no cover
@@ -495,7 +403,7 @@ def main():  # pragma: no cover
     configure_signals()
 
     try:
-        args = parse_arguments(*sys.argv[1:])
+        arguments = parse_arguments(*sys.argv[1:])
     except ValueError as error:
         configure_logging(logging.CRITICAL)
         logger.critical(error)
@@ -507,25 +415,32 @@ def main():  # pragma: no cover
         logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv)))
         exit_with_help_link()
 
-    colorama.init(autoreset=True, strip=not should_do_markup(args.no_color))
-
-    configure_logging(
-        verbosity_to_log_level(args.verbosity), verbosity_to_log_level(args.syslog_verbosity)
-    )
-
-    if args.version:
+    global_arguments = arguments['global']
+    if global_arguments.version:
         print(pkg_resources.require('borgmatic')[0].version)
         sys.exit(0)
 
-    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
+    configs, parse_logs = load_configurations(config_filenames)
+
+    colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
+    configure_logging(
+        verbosity_to_log_level(global_arguments.verbosity),
+        verbosity_to_log_level(global_arguments.syslog_verbosity),
+    )
+
     logger.debug('Ensuring legacy configuration is upgraded')
     convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
 
-    summary_logs = tuple(collect_configuration_run_summary_logs(config_filenames, args))
+    summary_logs = list(collect_configuration_run_summary_logs(configs, arguments))
 
     logger.info('')
     logger.info('summary:')
-    [logger.handle(log) for log in summary_logs if log.levelno >= logger.getEffectiveLevel()]
+    [
+        logger.handle(log)
+        for log in parse_logs + summary_logs
+        if log.levelno >= logger.getEffectiveLevel()
+    ]
 
     if any(log.levelno == logging.CRITICAL for log in summary_logs):
         exit_with_help_link()

borgmatic/commands/validate_config.py

@@ -3,9 +3,8 @@ import sys
 from argparse import ArgumentParser
 
 from borgmatic.config import collect, validate
-from borgmatic.logger import get_logger
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def parse_arguments(*arguments):

borgmatic/config/generate.py

@@ -45,10 +45,10 @@ def _comment_out_line(line):
     # Comment out the names of optional sections.
     one_indent = ' ' * INDENT
     if not line.startswith(one_indent):
-        return '#' + line
+        return '# ' + line
 
     # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
-    return '#'.join((one_indent, line[INDENT:]))
+    return '# '.join((one_indent, line[INDENT:]))
 
 
 REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}

borgmatic/config/load.py

@@ -1,10 +1,9 @@
+import logging
 import os
 
 import ruamel.yaml
 
-from borgmatic.logger import get_logger
-
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def load_configuration(filename):
@@ -54,9 +53,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
 
         for index, (key_node, value_node) in enumerate(node.value):
             if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
-                included_value = representer.represent_mapping(
-                    tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
-                )
+                included_value = representer.represent_data(self.construct_object(value_node))
                 node.value[index] = (key_node, included_value)
 
         super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -11,7 +11,7 @@ map:
             source_directories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     List of source directories to backup (required). Globs and tildes are expanded.
                 example:
@@ -21,7 +21,7 @@ map:
             repositories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Paths to local or remote repositories (required). Tildes are expanded. Multiple
                     repositories are backed up to in sequence. See ssh_command for SSH options like
@@ -36,6 +36,18 @@ map:
                 type: bool
                 desc: Only store/extract numeric user and group identifiers. Defaults to false.
                 example: true
+            atime:
+                type: bool
+                desc: Store atime into archive. Defaults to true.
+                example: false
+            ctime:
+                type: bool
+                desc: Store ctime into archive. Defaults to true.
+                example: false
+            birthtime:
+                type: bool
+                desc: Store birthtime (creation date) into archive. Defaults to true.
+                example: false
             read_special:
                 type: bool
                 desc: |
@@ -48,23 +60,23 @@ map:
                 desc: Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true.
                 example: true
             files_cache:
-                type: scalar
+                type: str
                 desc: |
                     Mode in which to operate the files cache. See
                     https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
                     details. Defaults to "ctime,size,inode".
                 example: ctime,size,inode
             local_path:
-                type: scalar
+                type: str
                 desc: Alternate Borg local executable. Defaults to "borg".
                 example: borg1
             remote_path:
-                type: scalar
+                type: str
                 desc: Alternate Borg remote executable. Defaults to "borg".
                 example: borg1
             patterns:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Any paths matching these patterns are included/excluded from backups. Globs are
                     expanded. (Tildes are not.) Note that Borg considers this option experimental.
@@ -77,7 +89,7 @@ map:
                     - '- /home/*'
             patterns_from:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Read include/exclude patterns from one or more separate named files, one pattern
                     per line. Note that Borg considers this option experimental. See the output of
@@ -86,7 +98,7 @@ map:
                     - /etc/borgmatic/patterns
             exclude_patterns:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Any paths matching these patterns are excluded from backups. Globs and tildes
                     are expanded. See the output of "borg help patterns" for more details.
@@ -96,7 +108,7 @@ map:
                     - /etc/ssl
             exclude_from:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Read exclude patterns from one or more separate named files, one pattern per
                     line. See the output of "borg help patterns" for more details.
@@ -109,7 +121,7 @@ map:
                     http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false.
                 example: true
             exclude_if_present:
-                type: scalar
+                type: str
                 desc: |
                     Exclude directories that contain a file with the given filename. Defaults to not
                     set.
@@ -122,7 +134,7 @@ map:
             details.
         map:
             encryption_passcommand:
-                type: scalar
+                type: str
                 desc: |
                     The standard output of this command is used to unlock the encryption key. Only
                     use on repositories that were initialized with passcommand/repokey encryption.
@@ -130,7 +142,7 @@ map:
                     then encryption_passphrase takes precedence. Defaults to not set.
                 example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
-                type: scalar
+                type: str
                 desc: |
                     Passphrase to unlock the encryption key with. Only use on repositories that were
                     initialized with passphrase/repokey encryption. Quote the value if it contains
@@ -145,14 +157,14 @@ map:
                     for details. Defaults to checkpoints every 1800 seconds (30 minutes).
                 example: 1800
             chunker_params:
-                type: scalar
+                type: str
                 desc: |
                     Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
                     HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
                     for details. Defaults to "19,23,21,4095".
                 example: 19,23,21,4095
             compression:
-                type: scalar
+                type: str
                 desc: |
                     Type of compression to use when creating archives. See
                     https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
@@ -163,34 +175,34 @@ map:
                 desc: Remote network upload rate limit in kiBytes/second. Defaults to unlimited.
                 example: 100
             ssh_command:
-                type: scalar
+                type: str
                 desc: |
                     Command to use instead of "ssh". This can be used to specify ssh options.
                     Defaults to not set.
                 example: ssh -i /path/to/private/key
             borg_base_directory:
-                type: scalar
+                type: str
                 desc: |
                     Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
                     See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
                 example: /path/to/base
             borg_config_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
                 example: /path/to/base/config
             borg_cache_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
                 example: /path/to/base/cache
             borg_security_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
                 example: /path/to/base/config/security
             borg_keys_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
                 example: /path/to/base/config/keys
@@ -203,7 +215,7 @@ map:
                 desc: Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1.
                 example: 5
             archive_name_format:
-                type: scalar
+                type: str
                 desc: |
                     Name of the archive. Borg placeholders can be used. See the output of
                     "borg help placeholders" for details. Defaults to
@@ -212,6 +224,16 @@ map:
                     archives with a different archive name format. And you should also specify a
                     prefix in the consistency section as well.
                 example: "{hostname}-documents-{now}"
+            relocated_repo_access_is_ok:
+                type: bool
+                desc: Bypass Borg error about a repository that has been moved. Defaults to false.
+                example: true
+            unknown_unencrypted_repo_access_is_ok:
+                type: bool
+                desc: |
+                    Bypass Borg error about a previously unknown unencrypted repository. Defaults to
+                    false.
+                example: true
     retention:
         desc: |
             Retention policy for how many backups to keep in each category. See
@@ -221,7 +243,7 @@ map:
             if you'd like to skip pruning entirely.
         map:
             keep_within:
-                type: scalar
+                type: str
                 desc: Keep all archives within this time interval.
                 example: 3H
             keep_secondly:
@@ -253,11 +275,11 @@ map:
                 desc: Number of yearly archives to keep.
                 example: 1
             prefix:
-                type: scalar
+                type: str
                 desc: |
                     When pruning, only consider archive names starting with this prefix.
                     Borg placeholders can be used. See the output of "borg help placeholders" for
-                    details. Defaults to "{hostname}-".
+                    details. Defaults to "{hostname}-". Use an empty value to disable the default.
                 example: sourcehostname
     consistency:
         desc: |
@@ -268,20 +290,21 @@ map:
             checks:
                 seq:
                     - type: str
-                      enum: ['repository', 'archives', 'extract', 'disabled']
+                      enum: ['repository', 'archives', 'data', 'extract', 'disabled']
                       unique: true
                 desc: |
-                    List of one or more consistency checks to run: "repository", "archives", and/or
-                    "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
-                    all consistency checks. "repository" checks the consistency of the repository,
-                    "archive" checks all of the archives, and "extract" does an extraction dry-run
-                    of the most recent archive.
+                    List of one or more consistency checks to run: "repository", "archives", "data",
+                    and/or "extract". Defaults to "repository" and "archives". Set to "disabled" to
+                    disable all consistency checks. "repository" checks the consistency of the
+                    repository, "archives" checks all of the archives, "data" verifies the integrity
+                    of the data within the archives, and "extract" does an extraction dry-run of the
+                    most recent archive. Note that "data" implies "archives".
                 example:
                     - repository
                     - archives
             check_repositories:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Paths to a subset of the repositories in the location section on which to run
                     consistency checks. Handy in case some of your repositories are very large, and
@@ -295,37 +318,78 @@ map:
                       "archives" check. Defaults to checking all archives.
                 example: 3
             prefix:
-                type: scalar
+                type: str
                 desc: |
                     When performing the "archives" check, only consider archive names starting with
                     this prefix. Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. Defaults to "{hostname}-".
+                    "borg help placeholders" for details. Defaults to "{hostname}-". Use an empty
+                    value to disable the default.
                 example: sourcehostname
+    output:
+        desc: |
+            Options for customizing borgmatic's own output and logging.
+        map:
+            color:
+                type: bool
+                desc: |
+                    Apply color to console output. Can be overridden with --no-color command-line
+                    flag. Defaults to true.
+                example: false
     hooks:
         desc: |
-            Shell commands or scripts to execute before and after a backup or if an error has occurred.
-            IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
-            Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
-            prevent potential shell injection or privilege escalation.
+            Shell commands, scripts, or integrations to execute at various points during a borgmatic
+            run. IMPORTANT: All provided commands and scripts are executed with user permissions of
+            borgmatic. Do not forget to set secure permissions on this configuration file (chmod
+            0600) as well as on any script called from a hook (chmod 0700) to prevent potential
+            shell injection or privilege escalation.
         map:
             before_backup:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute before creating a backup.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute before creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "Starting a backup job."
+                    - echo "Starting a backup."
             after_backup:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute after creating a backup.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute after creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "Backup created."
+                    - echo "Created a backup."
             on_error:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute when an exception
+                    occurs during a backup or when running a before_backup or after_backup hook.
                 example:
-                    - echo "Error while creating a backup."
+                    - echo "Error while creating a backup or running a backup hook."
+            healthchecks:
+                type: str
+                desc: |
+                    Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
+                    Create an account at https://healthchecks.io if you'd like to use this service.
+                example:
+                    https://hc-ping.com/your-uuid-here
+            before_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute before running all
+                    actions (if one of them is "create"), run once before all configuration files.
+                example:
+                    - echo "Starting actions."
+            after_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute after running all
+                    actions (if one of them is "create"), run once after all configuration files.
+                example:
+                    - echo "Completed actions."
             umask:
                 type: scalar
                 desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.

borgmatic/config/validate.py

@@ -6,9 +6,6 @@ import pykwalify.errors
 import ruamel.yaml
 
 from borgmatic.config import load
-from borgmatic.logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def schema_filename():
@@ -66,13 +63,6 @@ def apply_logical_validation(config_filename, parsed_configuration):
                 ),
             )
 
-    consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
-    if archive_name_format and not consistency_prefix:
-        logger.warning(
-            'Since version 1.1.16, if you provide `archive_name_format`, you should also'
-            ' specify `consistency.prefix`.'
-        )
-
 
 def parse_configuration(config_filename, schema_filename):
     '''

borgmatic/execute.py

@@ -1,12 +1,15 @@
 import logging
 import subprocess
 
-from borgmatic.logger import get_logger
+logger = logging.getLogger(__name__)
 
-logger = get_logger(__name__)
+
+ERROR_OUTPUT_MAX_LINE_COUNT = 25
+BORG_ERROR_EXIT_CODE = 2
 
 
 def execute_and_log_output(full_command, output_log_level, shell):
+    last_lines = []
     process = subprocess.Popen(
         full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
     )
@@ -16,18 +19,33 @@ def execute_and_log_output(full_command, output_log_level, shell):
         if not line:
             continue
 
-        if line.startswith('borg: error:'):
-            logger.error(line)
-        else:
-            logger.log(output_log_level, line)
+        # Keep the last few lines of output in case the command errors, and we need the output for
+        # the exception below.
+        last_lines.append(line)
+        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.pop(0)
+
+        logger.log(output_log_level, line)
 
     remaining_output = process.stdout.read().rstrip().decode()
     if remaining_output:  # pragma: no cover
         logger.log(output_log_level, remaining_output)
 
     exit_code = process.poll()
-    if exit_code != 0:
-        raise subprocess.CalledProcessError(exit_code, full_command)
+
+    # If shell is True, assume we're running something other than Borg and should treat all non-zero
+    # exit codes as errors.
+    error = bool(exit_code != 0) if shell else bool(exit_code >= BORG_ERROR_EXIT_CODE)
+
+    if error:
+        # If an error occurs, include its output in the raised exception so that we don't
+        # inadvertently hide error output.
+        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.insert(0, '...')
+
+        raise subprocess.CalledProcessError(
+            exit_code, ' '.join(full_command), '\n'.join(last_lines)
+        )
 
 
 def execute_command(full_command, output_log_level=logging.INFO, shell=False):
@@ -35,6 +53,8 @@ def execute_command(full_command, output_log_level=logging.INFO, shell=False):
     Execute the given command (a sequence of command/argument strings) and log its output at the
     given log level. If output log level is None, instead capture and return the output. If
     shell is True, execute the command within a shell.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
     logger.debug(' '.join(full_command))
 
@@ -43,3 +63,18 @@ def execute_command(full_command, output_log_level=logging.INFO, shell=False):
         return output.decode() if output is not None else None
     else:
         execute_and_log_output(full_command, output_log_level, shell=shell)
+
+
+def execute_command_without_capture(full_command):
+    '''
+    Execute the given command (a sequence of command/argument strings), but don't capture or log its
+    output in any way. This is necessary for commands that monkey with the terminal (e.g. progress
+    display) or provide interactive prompts.
+    '''
+    logger.debug(' '.join(full_command))
+
+    try:
+        subprocess.check_call(full_command)
+    except subprocess.CalledProcessError as error:
+        if error.returncode >= BORG_ERROR_EXIT_CODE:
+            raise

borgmatic/hook.py

@@ -1,19 +1,35 @@
 import logging
 import os
 
+import requests
+
 from borgmatic import execute
-from borgmatic.logger import get_logger
 
-logger = get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
-def execute_hook(commands, umask, config_filename, description, dry_run):
+def interpolate_context(command, context):
+    '''
+    Given a single hook command and a dict of context names/values, interpolate the values by
+    "{name}" into the command and return the result.
+    '''
+    for name, value in context.items():
+        command = command.replace('{%s}' % name, str(value))
+
+    return command
+
+
+def execute_hook(commands, umask, config_filename, description, dry_run, **context):
     '''
     Given a list of hook commands to execute, a umask to execute with (or None), a config filename,
     a hook description, and whether this is a dry run, run the given commands. Or, don't run them
     if this is a dry run.
 
+    The context contains optional values interpolated by name into the hook commands. Currently,
+    this only applies to the on_error hook.
+
     Raise ValueError if the umask cannot be parsed.
+    Raise subprocesses.CalledProcessError if an error occurs in a hook.
     '''
     if not commands:
         logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
@@ -21,6 +37,9 @@ def execute_hook(commands, umask, config_filename, description, dry_run):
 
     dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
 
+    context['configuration_filename'] = config_filename
+    commands = [interpolate_context(command, context) for command in commands]
+
     if len(commands) == 1:
         logger.info(
             '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
@@ -52,3 +71,34 @@ def execute_hook(commands, umask, config_filename, description, dry_run):
     finally:
         if original_umask:
             os.umask(original_umask)
+
+
+def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
+    '''
+    Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given
+    configuration filename in any log entries. If this is a dry run, then don't actually ping
+    anything.
+    '''
+    if not ping_url_or_uuid:
+        logger.debug('{}: No healthchecks hook set'.format(config_filename))
+        return
+
+    ping_url = (
+        ping_url_or_uuid
+        if ping_url_or_uuid.startswith('http')
+        else 'https://hc-ping.com/{}'.format(ping_url_or_uuid)
+    )
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+    if append:
+        ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info(
+        '{}: Pinging healthchecks.io{}{}'.format(
+            config_filename, ' ' + append if append else '', dry_run_label
+        )
+    )
+    logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url))
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    requests.get(ping_url)

borgmatic/logger.py

@@ -21,19 +21,31 @@ def to_bool(arg):
     return False
 
 
-def should_do_markup(no_color):
+def interactive_console():
     '''
-    Determine if we should enable colorama marking up.
+    Return whether the current console is "interactive". Meaning: Capable of
+    user input and not just something like a cron job.
+    '''
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+
+
+def should_do_markup(no_color, configs):
+    '''
+    Given the value of the command-line no-color argument, and a dict of configuration filename to
+    corresponding parsed configuration, determine if we should enable colorama marking up.
     '''
     if no_color:
         return False
 
+    if any(config.get('output', {}).get('color') is False for config in configs.values()):
+        return False
+
     py_colors = os.environ.get('PY_COLORS', None)
 
     if py_colors is not None:
         return to_bool(py_colors)
 
-    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+    return interactive_console()
 
 
 LOG_LEVEL_TO_COLOR = {
@@ -45,52 +57,10 @@ LOG_LEVEL_TO_COLOR = {
 }
 
 
-class Borgmatic_logger(logging.Logger):
-    def critical(self, msg, *args, **kwargs):
-        color = LOG_LEVEL_TO_COLOR.get(logging.CRITICAL)
-
-        return super(Borgmatic_logger, self).critical(color_text(color, msg), *args, **kwargs)
-
-    def error(self, msg, *args, **kwargs):
-        color = LOG_LEVEL_TO_COLOR.get(logging.ERROR)
-
-        return super(Borgmatic_logger, self).error(color_text(color, msg), *args, **kwargs)
-
-    def warn(self, msg, *args, **kwargs):
-        color = LOG_LEVEL_TO_COLOR.get(logging.WARN)
-
-        return super(Borgmatic_logger, self).warn(color_text(color, msg), *args, **kwargs)
-
-    def info(self, msg, *args, **kwargs):
-        color = LOG_LEVEL_TO_COLOR.get(logging.INFO)
-
-        return super(Borgmatic_logger, self).info(color_text(color, msg), *args, **kwargs)
-
-    def debug(self, msg, *args, **kwargs):
-        color = LOG_LEVEL_TO_COLOR.get(logging.DEBUG)
-
-        return super(Borgmatic_logger, self).debug(color_text(color, msg), *args, **kwargs)
-
-    def handle(self, record):
+class Console_color_formatter(logging.Formatter):
+    def format(self, record):
         color = LOG_LEVEL_TO_COLOR.get(record.levelno)
-        colored_record = logging.makeLogRecord(
-            dict(
-                levelno=record.levelno,
-                levelname=record.levelname,
-                msg=color_text(color, record.msg),
-            )
-        )
-
-        return super(Borgmatic_logger, self).handle(colored_record)
-
-
-def get_logger(name=None):
-    '''
-    Build a logger with the given name.
-    '''
-    logging.setLoggerClass(Borgmatic_logger)
-    logger = logging.getLogger(name)
-    return logger
+        return color_text(color, record.msg)
 
 
 def color_text(color, message):
@@ -111,7 +81,7 @@ def configure_logging(console_log_level, syslog_log_level=None):
         syslog_log_level = console_log_level
 
     console_handler = logging.StreamHandler()
-    console_handler.setFormatter(logging.Formatter('%(message)s'))
+    console_handler.setFormatter(Console_color_formatter())
     console_handler.setLevel(console_log_level)
 
     syslog_path = None
@@ -120,9 +90,9 @@ def configure_logging(console_log_level, syslog_log_level=None):
     elif os.path.exists('/var/run/syslog'):
         syslog_path = '/var/run/syslog'
 
-    if syslog_path:
+    if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
-        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s \ufeff%(message)s'))
+        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
     else:

docs/Dockerfile

@@ -1,10 +1,15 @@
-FROM python:3.7.3-alpine3.9 as borgmatic
+FROM python:3.7.4-alpine3.10 as borgmatic
 
 COPY . /app
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
-RUN borgmatic --help > /command-line.txt
+RUN borgmatic --help > /command-line.txt \
+    && for action in init prune create check extract list info; do \
+           echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
+           && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:11.15.0-alpine as html
+FROM node:12.10.0-alpine as html
+
+ARG ENVIRONMENT=production
 
 WORKDIR /source
 
@@ -17,10 +22,10 @@ RUN npm install @11ty/eleventy \
 COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml
 COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt
 COPY . /source
-RUN npx eleventy --input=/source/docs --output=/output/docs \
+RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.16.0-alpine
+FROM nginx:1.16.1-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/_includes/components/suggestion-form.css

@@ -0,0 +1,18 @@
+#suggestion-form textarea {
+	font-family: sans-serif;
+	width: 100%;
+}
+
+#suggestion-form label {
+	font-weight: bold;
+}
+
+#suggestion-form input[type=email] {
+	font-size: 16px;
+	width: 100%;
+}
+
+#suggestion-form .form-error {
+        color: red;
+}
+

docs/_includes/components/suggestion-form.html

@@ -0,0 +1,33 @@
+<h2>Improve this documentation</h2>
+
+<p>Have an idea on how to make this documentation even better? Send your
+feedback below! (But if you need help installing or using borgmatic, please
+use our <a href="https://torsion.org/borgmatic/#issues">issue tracker</a>
+instead.)</p>
+
+<form id="suggestion-form">
+    <div><label for="suggestion">Suggestion</label></div>
+    <textarea id="suggestion" rows="8" cols="60" name="suggestion"></textarea>
+    <div data-sk-error="suggestion" class="form-error"></div>
+    <input id="_page" type="hidden" name="_page">
+    <input id="_subject" type="hidden" name="_subject" value="borgmatic documentation suggestion">
+    <br />
+    <label for="email">Email address</label>
+    <div><input id="email" type="email" name="email" placeholder="Only required if you want a response!"></div>
+    <div data-sk-error="email" class="form-error"></div>
+    <br />
+    <div><button type="submit">Send</button></div>
+    <br />
+</form>
+
+<script>
+    document.getElementById('_page').value = window.location.href;
+    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
+  
+    sk('form', 'init', {
+        id: '1d536680ab96',
+        element: '#suggestion-form'
+    });
+</script>
+
+<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/layouts/base.njk

@@ -11,6 +11,7 @@
 {% include 'components/minilink.css' %}
 {% include 'components/toc.css' %}
 {% include 'components/info-blocks.css' %}
+{% include 'components/suggestion-form.css' %}
 {% include 'prism-theme.css' %}
 {% include 'asciinema.css' %}
 {% endset %}

docs/_includes/layouts/main.njk

@@ -8,5 +8,7 @@ headerClass: elv-header-default
 <main class="elv-layout{% if layoutClass %} {{ layoutClass }}{% endif %}">
     <article>
         {{ content | safe }}
+
+        {% include 'components/suggestion-form.html' %}
     </article>
 </main>

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -0,0 +1,76 @@
+---
+title: Add preparation and cleanup steps to backups
+---
+## Preparation and cleanup hooks
+
+If you find yourself performing prepraration tasks before your backup runs, or
+cleanup work afterwards, borgmatic hooks may be of interest. Hooks are
+shell commands that borgmatic executes for you at various points, and they're
+configured in the `hooks` section of your configuration file.
+
+For instance, you can specify `before_backup` hooks to dump a database to file
+before backing it up, and specify `after_backup` hooks to delete the temporary
+file afterwards. Here's an example:
+
+```yaml
+hooks:
+    before_backup:
+        - dump-a-database /to/file.sql
+    after_backup:
+        - rm /to/file.sql
+```
+
+The `before_backup` and `after_backup` hooks each run once per configuration
+file. `before_backup` hooks run prior to backups of all repositories in a
+configuration file, right before the `create` action. `after_backup` hooks run
+afterwards, but not if an error occurs in a previous hook or in the backups
+themselves.
+
+You can also use `before_everything` and `after_everything` hooks to perform
+global setup or cleanup:
+
+```yaml
+hooks:
+    before_everything:
+        - set-up-stuff-globally
+    after_everything:
+        - clean-up-stuff-globally
+```
+
+`before_everything` hooks collected from all borgmatic configuration files run
+once before all configuration files (prior to all actions), but only if there
+is a `create` action. An error encountered during a `before_everything` hook
+causes borgmatic to exit without creating backups.
+
+`after_everything` hooks run once after all configuration files and actions,
+but only if there is a `create` action. It runs even if an error occurs during
+a backup or a backup hook, but not if an error occurs during a
+`before_everything` hook.
+
+borgmatic also runs `on_error` hooks if an error occurs, either when creating
+a backup or running a backup hook. See the [monitoring and alerting
+documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups.md)
+for more information.
+
+## Hook output
+
+Any output produced by your hooks shows up both at the console and in syslog
+(when run in a non-interactive console). For more information, read about <a
+href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md">inspecting
+your backups</a>.
+
+## Security
+
+An important security note about hooks: borgmatic executes all hook commands
+with the user permissions of borgmatic itself. So to prevent potential shell
+injection or privilege escalation, do not forget to set secure permissions
+on borgmatic configuration files (`chmod 0600`) and scripts (`chmod 0700`)
+invoked by hooks.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups.md)

docs/how-to/deal-with-very-large-backups.md

@@ -14,18 +14,21 @@ repositories.
 
 If you find yourself in this situation, you have some options. First, you can
 run borgmatic's pruning, creating, or checking actions separately. For
-instance, the the following optional flags are available:
+instance, the the following optional actions are available:
 
 ```bash
-borgmatic --prune
-borgmatic --create
-borgmatic --check
+borgmatic prune
+borgmatic create
+borgmatic check
 ```
 
-You can run with only one of these flags provided, or you can mix and match
+(No borgmatic `prune`, `create`, or `check` actions? Try the old-style
+`--prune`, `--create`, or `--check`. Or upgrade borgmatic!)
+
+You can run with only one of these actions provided, or you can mix and match
 any number of them in a single borgmatic run. This supports approaches like
-making backups with `--create` on a frequent schedule, while only running
-expensive consistency checks with `--check` on a much less frequent basis from
+making backups with `create` on a frequent schedule, while only running
+expensive consistency checks with `check` on a much less frequent basis from
 a separate cron job.
 
 ### Consistency check configuration
@@ -65,6 +68,16 @@ consistency:
         - path/of/repository_to_check.borg
 ```
 
+Finally, you can override your configuration file's consistency checks, and
+run particular checks via the command-line. For instance:
+
+```bash
+borgmatic check --only data --only extract
+```
+
+This is useful for running slow consistency checks on an infrequent basis,
+separate from your regular checks.
+
 
 ## Troubleshooting
 

docs/how-to/inspect-your-backups.md

@@ -20,33 +20,48 @@ Or, for even more progress and debug spew:
 borgmatic --verbosity 2
 ```
 
+## Backup summary
+
+If you're less concerned with progress during a backup, and you only want to
+see the summary of archive statistics at the end, you can use the stats
+option when performing a backup:
+
+```bash
+borgmatic --stats
+```
+
 ## Existing backups
 
-Borgmatic provides convenient flags for Borg's
+borgmatic provides convenient actions for Borg's
 [list](https://borgbackup.readthedocs.io/en/stable/usage/list.html) and
 [info](https://borgbackup.readthedocs.io/en/stable/usage/info.html)
 functionality:
 
 
 ```bash
-borgmatic --list
-borgmatic --info
+borgmatic list
+borgmatic info
 ```
 
+(No borgmatic `list` or `info` actions? Try the old-style `--list` or
+`--info`. Or upgrade borgmatic!)
+
+
 ## Logging
 
 By default, borgmatic logs to a local syslog-compatible daemon if one is
-present. Where those logs show up depends on your particular system. If you're
-using systemd, try running `journalctl -xe`. Otherwise, try viewing
-`/var/log/syslog` or similiar.
+present and borgmatic is running in a non-interactive console. Where those
+logs show up depends on your particular system. If you're using systemd, try
+running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or
+similiar.
 
 You can customize the log level used for syslog logging with the
 `--syslog-verbosity` flag, and this is independent from the console logging
-`--verbosity` flag described above. For instance, to disable syslog logging
-except for errors:
+`--verbosity` flag described above. For instance, to get additional
+information about the progress of the backup as it proceeds:
 
 ```bash
-borgmatic --syslog-verbosity 0
+borgmatic --syslog-verbosity 1
 ```
 
 Or to increase syslog logging to include debug spew:
@@ -68,18 +83,10 @@ Note that the [sample borgmatic systemd service
 file](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#systemd)
 already has this rate limit disabled.
 
-## Scripting borgmatic
-
-To consume the output of borgmatic in other software, you can include an
-optional `--json` flag with `--create`, `--list`, or `--info` to get the
-output formatted as JSON.
-
-Note that when you specify the `--json` flag, Borg's other non-JSON output is
-suppressed so as not to interfere with the captured JSON. Also note that JSON
-output only shows up at the console, and not in syslog.
-
 
 ## Related documentation
 
  * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups.md)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md)
  * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)

docs/how-to/make-per-application-backups.md

@@ -105,6 +105,10 @@ include, the local file's option takes precedent. And note that this is a
 shallow merge rather than a deep merge, so the merging does not descend into
 nested values.
 
+Note that this `<<` include merging syntax is only for merging in mappings
+(keys/values). If you'd like to include other types like scalars or lists
+directly, please see the section above about standard includes.
+
 
 ## Related documentation
 

docs/how-to/monitor-your-backups.md

@@ -0,0 +1,158 @@
+---
+title: How to monitor your backups
+---
+
+## Monitoring and alerting
+
+Having backups is great, but they won't do you a lot of good unless you have
+confidence that they're running on a regular basis. That's where monitoring
+and alerting comes in.
+
+There are several different ways you can monitor your backups and find out
+whether they're succeeding. Which of these you choose to do is up to you and
+your particular infrastructure:
+
+1. **Job runner alerts**: The easiest place to start is with failure alerts
+from the [scheduled job
+runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) (cron,
+systemd, etc.) that's running borgmatic. But note that if the job doesn't even
+get scheduled (e.g. due to the job runner not running), you probably won't get
+an alert at all! Still, this is a decent first line of defense, especially
+when combined with some of the other approaches below.
+2. **borgmatic error hooks**: The `on_error` hook allows you to run an arbitrary
+command or script when borgmatic itself encounters an error running your
+backups. So for instance, you can run a script to send yourself a text message
+alert. But note that if borgmatic doesn't actually run, this alert won't fire.
+See [error
+hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
+below for how to configure this.
+4. **borgmatic Healthchecks hook**: This feature integrates with the
+[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks
+whenever borgmatic runs. That way, Healthchecks can alert you when something
+goes wrong or it doesn't hear from borgmatic for a configured interval. See
+[Healthchecks
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
+below for how to configure this.
+3. **Third-party monitoring software**: You can use traditional monitoring
+software to consume borgmatic JSON output and track when the last
+successful backup occurred. See [scripting
+borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
+below for how to configure this.
+5. **Borg hosting providers**: Most [Borg hosting
+providers](https://torsion.org/borgmatic/#hosting-providers) include
+monitoring and alerting as part of their offering. This gives you a dashboard
+to check on all of your backups, and can alert you if the service doesn't hear
+from borgmatic for a configured interval.
+6. **borgmatic consistency checks**: While not strictly part of monitoring, if you
+really want confidence that your backups are not only running but are
+restorable as well, you can configure particular [consistency
+checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration)
+or even script full [restore
+tests](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/).
+
+
+## Error hooks
+
+When an error occurs during a backup, borgmatic can run configurable shell
+commands to fire off custom error notifications or take other actions, so you
+can get alerted as soon as something goes wrong. Here's a not-so-useful
+example:
+
+```yaml
+hooks:
+    on_error:
+        - echo "Error while creating a backup or running a backup hook."
+```
+
+The `on_error` hook supports interpolating particular runtime variables into
+the hook command. Here's an example that assumes you provide a separate shell
+script to handle the alerting:
+
+```yaml
+hooks:
+    on_error:
+        - send-text-message.sh "{configuration_filename}" "{repository}"
+```
+
+In this example, when the error occurs, borgmatic interpolates a few runtime
+values into the hook command: the borgmatic configuration filename, and the
+path of the repository. Here's the full set of supported variables you can use
+here:
+
+ * `configuration_filename`: borgmatic configuration filename in which the
+   error occurred
+ * `repository`: path of the repository in which the error occurred (may be
+   blank if the error occurs in a hook)
+ * `error`: the error message itself
+ * `output`: output of the command that failed (may be blank if an error
+   occurred without running a command)
+
+Note that borgmatic does not run `on_error` hooks if an error occurs within a
+`before_everything` or `after_everything` hook. For more about hooks, see the
+[borgmatic hooks
+documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md),
+especially the security information.
+
+
+## Healthchecks hook
+
+[Healthchecks](https://healthchecks.io/) is a service that provides "instant
+alerts when your cron jobs fail silently", and borgmatic has built-in
+integration with it. Once you create a Healthchecks account and project on
+their site, all you need to do is configure borgmatic with the unique "Ping
+URL" for your project. Here's an example:
+
+
+```yaml
+hooks:
+    healthchecks: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
+```
+
+With this hook in place, borgmatic will ping your Healthchecks project when a
+backup begins, ends, or errors. Then you can configure Healthchecks to notify
+you by a [variety of
+mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
+or it doesn't hear from borgmatic for a certain period of time.
+
+
+## Scripting borgmatic
+
+To consume the output of borgmatic in other software, you can include an
+optional `--json` flag with `create`, `list`, or `info` to get the output
+formatted as JSON.
+
+Note that when you specify the `--json` flag, Borg's other non-JSON output is
+suppressed so as not to interfere with the captured JSON. Also note that JSON
+output only shows up at the console, and not in syslog.
+
+
+### Successful backups
+
+`borgmatic list` includes support for a `--successful` flag that only lists
+successful (non-checkpoint) backups. This flag works via a basic heuristic: It
+assumes that non-checkpoint archive names end with a digit (e.g. from a
+timestamp), while checkpoint archive names do not. This means that if you're
+using custom archive names that do not end in a digit, the `--successful` flag
+will not work as expected.
+
+Combined with a built-in Borg flag like `--last`, you can list the last
+successful backup for use in your monitoring scripts. Here's an example
+combined with `--json`:
+
+```bash
+borgmatic list --successful --last 1 --json
+```
+
+Note that this particular combination will only work if you've got a single
+backup "series" in your repository. If you're instead backing up, say, from
+multiple different hosts into a single repository, then you'll need to get
+fancier with your archive listing. See `borg list --help` for more flags.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md)
+ * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup.md)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)

docs/how-to/restore-a-backup.md

@@ -5,12 +5,15 @@ title: How to restore a backup
 
 When the worst happens—or you want to test your backups—the first step is
 to figure out which archive to restore. A good way to do that is to use the
-`--list` action:
+`list` action:
 
 ```bash
-borgmatic --list
+borgmatic list
 ```
 
+(No borgmatic `list` action? Try the old-style `--list`, or upgrade
+borgmatic!)
+
 That should yield output looking something like:
 
 ```text
@@ -22,9 +25,12 @@ Assuming that you want to restore the archive with the most up-to-date files
 and therefore the latest timestamp, run a command like:
 
 ```bash
-borgmatic --extract --archive host-2019-01-02T04:06:07.080910
+borgmatic extract --archive host-2019-01-02T04:06:07.080910
 ```
 
+(No borgmatic `extract` action? Try the old-style `--extract`, or upgrade
+borgmatic!)
+
 The `--archive` value is the name of the archive to restore. This extracts the
 entire contents of the archive to the current directory, so make sure you're
 in the right place before running the command.
@@ -33,13 +39,13 @@ in the right place before running the command.
 ## Repository selection
 
 If you have a single repository in your borgmatic configuration file(s), no
-problem: the `--extract` action figures out which repository to use.
+problem: the `extract` action figures out which repository to use.
 
 But if you have multiple repositories configured, then you'll need to specify
 the repository path containing the archive to extract. Here's an example:
 
 ```bash
-borgmatic --extract --repository repo.borg --archive host-2019-...
+borgmatic extract --repository repo.borg --archive host-2019-...
 ```
 
 ## Restore particular files
@@ -49,7 +55,7 @@ everything from an archive. To do that, tack on one or more `--restore-path`
 values. For instance:
 
 ```bash
-borgmatic --extract --archive host-2019-... --restore-path /path/1 /path/2
+borgmatic extract --archive host-2019-... --restore-path /path/1 /path/2
 ```
 
 Like a whole-archive restore, this also restores into the current directory.
@@ -59,3 +65,4 @@ Like a whole-archive restore, this also restores into the current directory.
 
  * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
  * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups.md)

docs/how-to/run-preparation-steps-before-backups.md

@@ -1,58 +1,3 @@
----
-title: How to run preparation steps before backups
----
-## Preparation and cleanup hooks
-
-If you find yourself performing prepraration tasks before your backup runs, or
-cleanup work afterwards, borgmatic hooks may be of interest. Hooks are
-shell commands that borgmatic executes for you at various points, and they're
-configured in the `hooks` section of your configuration file.
-
-For instance, you can specify `before_backup` hooks to dump a database to file
-before backing it up, and specify `after_backup` hooks to delete the temporary
-file afterwards. Here's an example:
-
-```yaml
-hooks:
-    before_backup:
-        - dump-a-database /to/file.sql
-    after_backup:
-        - rm /to/file.sql
-```
-
-borgmatic hooks run once per configuration file. `before_backup` hooks run
-prior to backups of all repositories. `after_backup` hooks run afterwards, but
-not if an error occurs in a previous hook or in the backups themselves.
-
-
-## Error hooks
-
-borgmatic also runs `on_error` hooks if an error occurs. Here's an example
-configuration:
-
-```yaml
-hooks:
-    on_error:
-        - echo "Error while creating a backup."
-```
-
-## Hook output
-
-Any output produced by your hooks shows up both at the console and in syslog.
-For more information, read about <a
-href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md">inspecting
-your backups</a>.
-
-## Security
-
-An important security note about hooks: borgmatic executes all hook commands
-with the user permissions of borgmatic itself. So to prevent potential shell
-injection or privilege escalation, do not forget to set secure permissions
-(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
-
-
-## Related documentation
-
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
- * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
- * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+<head>
+    <meta http-equiv='refresh' content='0; URL=https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/'>
+</head>

docs/how-to/set-up-backups.md

@@ -4,14 +4,14 @@ title: How to set up backups with borgmatic
 ## Installation
 
 To get up and running, first [install
-Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
+Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at
 least version 1.1.
 
-Borgmatic consumes configurations in `/etc/borgmatic/` and `/etc/borgmatic.d/`
-by default. Therefore, we show how to install borgmatic for the root user which
-will have access permissions for these locations by default.
+By default, borgmatic looks for its configuration files in `/etc/borgmatic/`
+and `/etc/borgmatic.d/`, where the root user typically has read access.
 
-Run the following commands to download and install borgmatic:
+So, to download and install borgmatic as the root user, run the following
+commands:
 
 ```bash
 sudo pip3 install --user --upgrade borgmatic
@@ -35,9 +35,11 @@ borgmatic:
  * [Debian](https://tracker.debian.org/pkg/borgmatic)
  * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
  * [Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
- * [Arch Linux](https://aur.archlinux.org/packages/borgmatic/)
+ * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
+ * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
+ * [virtualenv](https://virtualenv.pypa.io/en/stable/)
 
 
 ## Hosting providers
@@ -71,7 +73,8 @@ to ignore anything you don't need.
 Note that the configuration file is organized into distinct sections, each
 with a section name like `location:` or `storage:`. So take care that if you
 uncomment a particular option, also uncomment its containing section name, or
-else borgmatic won't recognize the option.
+else borgmatic won't recognize the option. Also be sure to use spaces rather
+than tabs for indentation; YAML does not allow tabs.
 
 You can also get the same sample configuration file from the [configuration
 reference](https://torsion.org/borgmatic/docs/reference/configuration.md), the authoritative set of
@@ -86,7 +89,7 @@ encrypt your Borg repository with a passphrase instead of a key file, you'll
 either need to set the borgmatic `encryption_passphrase` configuration
 variable or set the `BORG_PASSPHRASE` environment variable. See the
 [repository encryption
-section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
+section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)
 of the Borg Quick Start for more info.
 
 Alternatively, you can specify the passphrase programatically by setting
@@ -121,18 +124,21 @@ this step if you already have a Borg repository.) To create a repository, run
 a command like the following:
 
 ```bash
-borgmatic --init --encryption repokey
+borgmatic init --encryption repokey
 ```
 
+(No borgmatic `init` action? Try the old-style `--init` flag, or upgrade
+borgmatic!)
+
 This uses the borgmatic configuration file you created above to determine
 which local or remote repository to create, and encrypts it with the
 encryption passphrase specified there if one is provided. Read about [Borg
 encryption
-modes](https://borgbackup.readthedocs.io/en/latest/usage/init.html#encryption-modes)
+modes](https://borgbackup.readthedocs.io/en/stable/usage/init.html#encryption-modes)
 for the menu of available encryption modes.
 
 Also, optionally check out the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) for more
+Start](https://borgbackup.readthedocs.org/en/stable/quickstart.html) for more
 background about repository initialization.
 
 Note that borgmatic skips repository initialization if the repository already
@@ -199,15 +205,32 @@ sudo systemctl start borgmatic.timer
 Feel free to modify the timer file based on how frequently you'd like
 borgmatic to run.
 
-## Colored Output
+## Colored output
 
-Borgmatic uses [colorama](https://pypi.org/project/colorama/) to produce
-colored terminal output by default. It is disabled when a non-interactive
-terminal is detected (like a cron job). Otherwise, it can be disabled by
-passing `--no-color` or by setting the environment variable `PY_COLORS=False`.
+Borgmatic produces colored terminal output by default. It is disabled when a
+non-interactive terminal is detected (like a cron job). Otherwise, you can
+disable it by passing the `--no-color` flag, setting the environment variable
+`PY_COLORS=False`, or setting the `color` option to `false` in the `output`
+section of configuration.
 
 ## Troubleshooting
 
+### "found character that cannot start any token" error
+
+If you run borgmatic and see an error looking something like this, it probably
+means you've used tabs instead of spaces:
+
+```
+test.yaml: Error parsing configuration file
+An error occurred while parsing a configuration file at config.yaml:
+while scanning for the next token
+found character that cannot start any token
+  in "config.yaml", line 230, column 1
+```
+
+YAML does not allow tabs. So to fix this, replace any tabs in your
+configuration file with the requisite number of spaces.
+
 ### libyaml compilation errors
 
 borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
@@ -224,10 +247,6 @@ it.
  * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
  * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups.md)
  * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups.md)
  * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration.md)
  * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line.md)
-
-<script>
-  var links = document.getElementsByClassName("referral");
-  links[Math.floor(Math.random() * links.length)].style.display = "none";
-</script>

docs/reference/command-line.md

@@ -3,7 +3,8 @@ title: borgmatic command-line reference
 ---
 ## borgmatic options
 
-Here are all of the available borgmatic command-line options:
+Here are all of the available borgmatic command-line options. This includes the separate options for
+each action sub-command:
 
 ```
 {% include borgmatic/command-line.txt %}

sample/cron-alpine/borgmatic

@@ -1,3 +0,0 @@
-# You can drop this file into /etc/cron.d/ to run borgmatic nightly.
-
-0 3 * * * PATH=$PATH:/usr/bin /root/.local/bin/borgmatic

sample/cron/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * root PATH=$PATH:/usr/local/bin /root/.local/bin/borgmatic
+0 3 * * * root PATH=$PATH:/usr/bin:/usr/local/bin /root/.local/bin/borgmatic --syslog-verbosity 1

sample/systemd/borgmatic.service

@@ -1,7 +1,22 @@
 [Unit]
 Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+ConditionACPower=true
 
 [Service]
 Type=oneshot
-ExecStart=/root/.local/bin/borgmatic
+
+# Lower CPU and I/O priority.
+Nice=19
+CPUSchedulingPolicy=batch
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+IOWeight=100
+
+Restart=no
 LogRateLimitIntervalSec=0
+
+# Delay start to prevent backups running during boot.
+ExecStartPre=sleep 1m
+ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --syslog-verbosity 1

scripts/dev-docs

@@ -2,5 +2,8 @@
 
 set -e
 
-docker build --tag borgmatic-docs --file docs/Dockerfile .
+docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile .
+echo
+echo "You can view dev docs at http://localhost:8080"
+echo
 docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs

scripts/find-unsupported-borg-options

@@ -18,7 +18,7 @@ mv temp.yaml.uncommented temp.yaml
 for sub_command in prune create check list info; do
     echo "********** borg $sub_command **********"
 
-    for line in $(borgmatic --config temp.yaml --$sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
+    for line in $(borgmatic --config temp.yaml $sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
         echo "$line" | grep '^-' >> borgmatic_borg_flags
     done
     sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
@@ -48,6 +48,17 @@ for sub_command in prune create check list info; do
             | grep -v '^--stats$' \
             | grep -v '^--verbose$' \
             | grep -v '^--warning$' \
+            | grep -v '^--exclude' \
+            | grep -v '^--exclude-from' \
+            | grep -v '^--first' \
+            | grep -v '^--format' \
+            | grep -v '^--glob-archives' \
+            | grep -v '^--last' \
+            | grep -v '^--list-format' \
+            | grep -v '^--patterns-from' \
+            | grep -v '^--prefix' \
+            | grep -v '^--short' \
+            | grep -v '^--sort-by' \
             | grep -v '^-h$' \
             >> all_borg_flags
     done

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.3.6'
+VERSION = '1.3.26'
 
 
 setup(
@@ -31,7 +31,8 @@ setup(
     obsoletes=['atticmatic'],
     install_requires=(
         'pykwalify>=1.6.0,<14.06',
-        'ruamel.yaml>0.15.0,<0.16.0',
+        'requests',
+        'ruamel.yaml>0.15.0,<0.17.0',
         'setuptools',
         'colorama>=0.4.1,<0.5',
     ),

test_requirements.txt

@@ -4,21 +4,22 @@ attrs==19.1.0
 black==19.3b0; python_version >= '3.6'
 click==7.0
 colorama==0.4.1
-coverage==4.5.3
+coverage==4.5.4
 docopt==0.6.2
-flake8==3.7.7
+flake8==3.7.8
 flexmock==0.10.4
-isort==4.3.20
+isort==4.3.21
 mccabe==0.6.1
-more-itertools==7.0.0
-pluggy==0.12.0
+more-itertools==7.2.0
+pluggy==0.13.0
 py==1.8.0
 pycodestyle==2.5.0
 pyflakes==2.1.1
 pykwalify==1.7.0
-pytest==4.6.3
+pytest==5.1.2
 pytest-cov==2.7.1
 python-dateutil==2.8.0
-PyYAML==5.1.1
-ruamel.yaml>0.15.0,<0.16.0
+PyYAML==5.1.2
+requests==2.22.0
+ruamel.yaml>0.15.0,<0.17.0
 toml==0.10.0

tests/integration/commands/test_arguments.py

@@ -0,0 +1,381 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.commands import arguments as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments()
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == ['myconfig', 'otherconfig']
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--verbosity', '1')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 1
+    assert global_arguments.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_syslog_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--syslog-verbosity', '2')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 2
+
+
+def test_parse_arguments_with_list_json_overrides_default():
+    arguments = module.parse_arguments('list', '--json')
+
+    assert 'list' in arguments
+    assert arguments['list'].json is True
+
+
+def test_parse_arguments_with_dashed_list_json_overrides_default():
+    arguments = module.parse_arguments('--list', '--json')
+
+    assert 'list' in arguments
+    assert arguments['list'].json is True
+
+
+def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments()
+
+    assert 'prune' in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--stats')
+
+    assert 'prune' in arguments
+    assert arguments['prune'].stats
+    assert 'create' in arguments
+    assert arguments['create'].stats
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_help_and_no_actions_shows_global_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'global arguments:' in captured.out
+    assert 'actions:' in captured.out
+
+
+def test_parse_arguments_with_help_and_action_shows_action_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('create', '--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'global arguments:' not in captured.out
+    assert 'actions:' not in captured.out
+    assert 'create arguments:' in captured.out
+
+
+def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('prune')
+
+    assert 'prune' in arguments
+    assert 'create' not in arguments
+    assert 'check' not in arguments
+
+
+def test_parse_arguments_with_dashed_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--prune')
+
+    assert 'prune' in arguments
+    assert 'create' not in arguments
+    assert 'check' not in arguments
+
+
+def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('create', 'check')
+
+    assert 'prune' not in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_multiple_dashed_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--create', '--check')
+
+    assert 'prune' not in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')
+
+
+def test_parse_arguments_disallows_deprecated_excludes_option():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+
+def test_parse_arguments_disallows_encryption_mode_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
+
+
+def test_parse_arguments_allows_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey')
+
+
+def test_parse_arguments_allows_encryption_mode_with_dashed_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey')
+
+
+def test_parse_arguments_requires_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', 'init')
+
+
+def test_parse_arguments_disallows_append_only_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--append-only')
+
+
+def test_parse_arguments_disallows_storage_quota_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
+
+
+def test_parse_arguments_allows_init_and_prune():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune')
+
+
+def test_parse_arguments_allows_init_and_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create')
+
+
+def test_parse_arguments_disallows_init_and_dry_run():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments(
+            '--config', 'myconfig', 'init', '--encryption', 'repokey', '--dry-run'
+        )
+
+
+def test_parse_arguments_disallows_glob_archives_with_successful():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments(
+            '--config', 'myconfig', 'list', '--glob-archives', '*glob*', '--successful'
+        )
+
+
+def test_parse_arguments_disallows_repository_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
+
+
+def test_parse_arguments_allows_repository_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments(
+        '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test'
+    )
+
+
+def test_parse_arguments_allows_repository_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg')
+
+
+def test_parse_arguments_disallows_archive_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--archive', 'test')
+
+
+def test_parse_arguments_disallows_restore_paths_without_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--restore-path', 'test')
+
+
+def test_parse_arguments_allows_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_dashed_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'list', '--archive', 'test')
+
+
+def test_parse_arguments_requires_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', 'extract')
+
+
+def test_parse_arguments_allows_progress_before_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', 'create', 'list')
+
+
+def test_parse_arguments_allows_progress_after_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('create', '--progress', 'list')
+
+
+def test_parse_arguments_allows_progress_and_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list')
+
+
+def test_parse_arguments_disallows_progress_without_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--progress', 'list')
+
+
+def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', 'create', 'list')
+
+
+def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', 'prune', 'list')
+
+
+def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--stats', 'list')
+
+
+def test_parse_arguments_allows_json_with_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('list', '--json')
+    module.parse_arguments('info', '--json')
+
+
+def test_parse_arguments_allows_json_with_dashed_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--info', '--json')
+
+
+def test_parse_arguments_disallows_json_with_both_list_and_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('list', 'info', '--json')
+
+
+def test_parse_arguments_check_only_extract_does_not_raise_extract_subparser_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('check', '--only', 'extract')
+
+
+def test_parse_arguments_extract_archive_check_does_not_raise_check_subparser_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('extract', '--archive', 'check')
+
+
+def test_parse_arguments_extract_with_check_only_extract_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract')

tests/integration/commands/test_borgmatic.py

@@ -1,281 +1,10 @@
 import subprocess
 
-import pytest
 from flexmock import flexmock
 
 from borgmatic.commands import borgmatic as module
 
 
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments()
-
-    assert parser.config_paths == config_paths
-    assert parser.excludes_filename is None
-    assert parser.verbosity == 0
-    assert parser.syslog_verbosity == 1
-    assert parser.json is False
-
-
-def test_parse_arguments_with_multiple_config_paths_parses_as_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
-
-    assert parser.config_paths == ['myconfig', 'otherconfig']
-    assert parser.verbosity == 0
-    assert parser.syslog_verbosity == 1
-
-
-def test_parse_arguments_with_verbosity_overrides_default():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments('--verbosity', '1')
-
-    assert parser.config_paths == config_paths
-    assert parser.excludes_filename is None
-    assert parser.verbosity == 1
-    assert parser.syslog_verbosity == 1
-
-
-def test_parse_arguments_with_syslog_verbosity_overrides_default():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments('--syslog-verbosity', '2')
-
-    assert parser.config_paths == config_paths
-    assert parser.excludes_filename is None
-    assert parser.verbosity == 0
-    assert parser.syslog_verbosity == 2
-
-
-def test_parse_arguments_with_json_overrides_default():
-    parser = module.parse_arguments('--list', '--json')
-    assert parser.json is True
-
-
-def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments()
-
-    assert parser.prune is True
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--prune')
-
-    assert parser.prune is True
-    assert parser.create is False
-    assert parser.check is False
-
-
-def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--create', '--check')
-
-    assert parser.prune is False
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_invalid_arguments_exits():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(SystemExit):
-        module.parse_arguments('--posix-me-harder')
-
-
-def test_parse_arguments_disallows_deprecated_excludes_option():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
-
-
-def test_parse_arguments_disallows_encryption_mode_without_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
-
-
-def test_parse_arguments_allows_encryption_mode_with_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey')
-
-
-def test_parse_arguments_requires_encryption_mode_with_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--init')
-
-
-def test_parse_arguments_disallows_append_only_without_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--append-only')
-
-
-def test_parse_arguments_disallows_storage_quota_without_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
-
-
-def test_parse_arguments_allows_init_and_prune():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--prune')
-
-
-def test_parse_arguments_allows_init_and_create():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--create')
-
-
-def test_parse_arguments_disallows_init_and_dry_run():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments(
-            '--config', 'myconfig', '--init', '--encryption', 'repokey', '--dry-run'
-        )
-
-
-def test_parse_arguments_disallows_repository_without_extract_or_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
-
-
-def test_parse_arguments_allows_repository_with_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments(
-        '--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
-    )
-
-
-def test_parse_arguments_allows_repository_with_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
-
-
-def test_parse_arguments_disallows_archive_without_extract_or_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--archive', 'test')
-
-
-def test_parse_arguments_disallows_restore_paths_without_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--restore-path', 'test')
-
-
-def test_parse_arguments_allows_archive_with_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
-
-
-def test_parse_arguments_allows_archive_with_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
-
-
-def test_parse_arguments_requires_archive_with_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--extract')
-
-
-def test_parse_arguments_allows_progress_and_create():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--progress', '--create', '--list')
-
-
-def test_parse_arguments_allows_progress_and_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
-
-
-def test_parse_arguments_disallows_progress_without_create():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--progress', '--list')
-
-
-def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--stats', '--create', '--list')
-
-
-def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--stats', '--prune', '--list')
-
-
-def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--stats', '--list')
-
-
-def test_parse_arguments_with_just_stats_flag_does_not_raise():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--stats')
-
-
-def test_parse_arguments_allows_json_with_list_or_info():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--list', '--json')
-    module.parse_arguments('--info', '--json')
-
-
-def test_parse_arguments_disallows_json_without_list_or_info():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--json')
-
-
-def test_parse_arguments_disallows_json_with_both_list_and_info():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--list', '--info', '--json')
-
-
 def test_borgmatic_version_matches_news_version():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/config/test_generate.py

@@ -31,17 +31,17 @@ def test_comment_out_line_skips_already_commented_out_line():
 def test_comment_out_line_comments_section_name():
     line = 'figgy-pudding:'
 
-    assert module._comment_out_line(line) == '#' + line
+    assert module._comment_out_line(line) == '# ' + line
 
 
 def test_comment_out_line_comments_indented_option():
     line = '    enabled: true'
 
-    assert module._comment_out_line(line) == '    #enabled: true'
+    assert module._comment_out_line(line) == '    # enabled: true'
 
 
 def test_comment_out_optional_configuration_comments_optional_config_only():
-    flexmock(module)._comment_out_line = lambda line: '#' + line
+    flexmock(module)._comment_out_line = lambda line: '# ' + line
     config = '''
 foo:
     bar:
@@ -56,18 +56,19 @@ location:
     other: thing
     '''
 
+    # flake8: noqa
     expected_config = '''
-#foo:
-#    bar:
-#        - baz
-#        - quux
-#
+# foo:
+#     bar:
+#         - baz
+#         - quux
+# 
 location:
     repositories:
         - one
         - two
-#
-#    other: thing
+# 
+#     other: thing
     '''
 
     assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()

tests/integration/config/test_load.py

@@ -1,5 +1,7 @@
 import sys
 
+import pytest
+import ruamel.yaml
 from flexmock import flexmock
 
 from borgmatic.config import load as module
@@ -38,3 +40,23 @@ def test_load_configuration_merges_include():
     )
 
     assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
+
+
+def test_load_configuration_does_not_merge_include_list():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+          - one
+          - two
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: bar
+        repositories:
+          <<: !include include.yaml
+        '''
+    )
+
+    with pytest.raises(ruamel.yaml.error.YAMLError):
+        assert module.load_configuration('config.yaml')

tests/integration/test_execute.py

@@ -15,12 +15,41 @@ def test_execute_and_log_output_logs_each_line_separately():
     module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False)
 
 
-def test_execute_and_log_output_logs_borg_error_as_error():
-    flexmock(module.logger).should_receive('error').with_args('borg: error: oopsie').once()
+def test_execute_and_log_output_with_borg_warning_does_not_raise():
+    flexmock(module.logger).should_receive('log')
 
-    module.execute_and_log_output(
-        ['echo', 'borg: error: oopsie'], output_log_level=logging.INFO, shell=False
-    )
+    # Borg's exit code 1 is a warning, not an error.
+    module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)
+
+
+def test_execute_and_log_output_includes_borg_error_output_in_exception():
+    flexmock(module.logger).should_receive('log')
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+
+    assert error.value.returncode == 2
+    assert error.value.output
+
+
+def test_execute_and_log_output_with_shell_error_raises():
+    flexmock(module.logger).should_receive('log')
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=True)
+
+    assert error.value.returncode == 1
+
+
+def test_execute_and_log_output_truncates_long_borg_error_output():
+    flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
+    flexmock(module.logger).should_receive('log')
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+
+    assert error.value.returncode == 2
+    assert error.value.output.startswith('...')
 
 
 def test_execute_and_log_output_with_no_output_logs_nothing():
@@ -30,7 +59,7 @@ def test_execute_and_log_output_with_no_output_logs_nothing():
 
 
 def test_execute_and_log_output_with_error_exit_status_raises():
-    flexmock(module.logger).should_receive('log').never()
+    flexmock(module.logger).should_receive('log')
 
     with pytest.raises(subprocess.CalledProcessError):
-        module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)

tests/unit/borg/test_check.py

@@ -34,32 +34,76 @@ def test_parse_checks_with_blank_value_returns_defaults():
     assert checks == module.DEFAULT_CHECKS
 
 
+def test_parse_checks_with_none_value_returns_defaults():
+    checks = module._parse_checks({'checks': None})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
 def test_parse_checks_with_disabled_returns_no_checks():
     checks = module._parse_checks({'checks': ['disabled']})
 
     assert checks == ()
 
 
+def test_parse_checks_with_data_check_also_injects_archives():
+    checks = module._parse_checks({'checks': ['data']})
+
+    assert checks == ('data', 'archives')
+
+
+def test_parse_checks_with_data_check_passes_through_archives():
+    checks = module._parse_checks({'checks': ['data', 'archives']})
+
+    assert checks == ('data', 'archives')
+
+
+def test_parse_checks_prefers_override_checks_to_configured_checks():
+    checks = module._parse_checks({'checks': ['archives']}, only_checks=['repository', 'extract'])
+
+    assert checks == ('repository', 'extract')
+
+
+def test_parse_checks_with_override_data_check_also_injects_archives():
+    checks = module._parse_checks({'checks': ['extract']}, only_checks=['data'])
+
+    assert checks == ('data', 'archives')
+
+
 def test_make_check_flags_with_repository_check_returns_flag():
     flags = module._make_check_flags(('repository',))
 
     assert flags == ('--repository-only',)
 
 
+def test_make_check_flags_with_archives_check_returns_flag():
+    flags = module._make_check_flags(('archives',))
+
+    assert flags == ('--archives-only',)
+
+
+def test_make_check_flags_with_data_check_returns_flag():
+    flags = module._make_check_flags(('data',))
+
+    assert flags == ('--verify-data',)
+
+
 def test_make_check_flags_with_extract_omits_extract_flag():
     flags = module._make_check_flags(('extract',))
 
     assert flags == ()
 
 
-def test_make_check_flags_with_default_checks_returns_default_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+def test_make_check_flags_with_default_checks_and_default_prefix_returns_default_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, prefix=module.DEFAULT_PREFIX)
 
     assert flags == ('--prefix', module.DEFAULT_PREFIX)
 
 
-def test_make_check_flags_with_all_checks_returns_default_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS + ('extract',))
+def test_make_check_flags_with_all_checks_and_default_prefix_returns_default_flags():
+    flags = module._make_check_flags(
+        module.DEFAULT_CHECKS + ('extract',), prefix=module.DEFAULT_PREFIX
+    )
 
     assert flags == ('--prefix', module.DEFAULT_PREFIX)
 
@@ -67,7 +111,7 @@ def test_make_check_flags_with_all_checks_returns_default_flags():
 def test_make_check_flags_with_archives_check_and_last_includes_last_flag():
     flags = module._make_check_flags(('archives',), check_last=3)
 
-    assert flags == ('--archives-only', '--last', '3', '--prefix', module.DEFAULT_PREFIX)
+    assert flags == ('--archives-only', '--last', '3')
 
 
 def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
@@ -79,7 +123,7 @@ def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
 def test_make_check_flags_with_default_checks_and_last_includes_last_flag():
     flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
 
-    assert flags == ('--last', '3', '--prefix', module.DEFAULT_PREFIX)
+    assert flags == ('--last', '3')
 
 
 def test_make_check_flags_with_archives_check_and_prefix_includes_prefix_flag():
@@ -88,6 +132,18 @@ def test_make_check_flags_with_archives_check_and_prefix_includes_prefix_flag():
     assert flags == ('--archives-only', '--prefix', 'foo-')
 
 
+def test_make_check_flags_with_archives_check_and_empty_prefix_omits_prefix_flag():
+    flags = module._make_check_flags(('archives',), prefix='')
+
+    assert flags == ('--archives-only',)
+
+
+def test_make_check_flags_with_archives_check_and_none_prefix_omits_prefix_flag():
+    flags = module._make_check_flags(('archives',), prefix=None)
+
+    assert flags == ('--archives-only',)
+
+
 def test_make_check_flags_with_repository_check_and_prefix_omits_prefix_flag():
     flags = module._make_check_flags(('repository',), prefix='foo-')
 
@@ -114,7 +170,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
     insert_execute_command_mock(('borg', 'check', 'repo'))
 
@@ -143,7 +199,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_logging_mock(logging.INFO)
-    insert_execute_command_mock(('borg', 'check', 'repo', '--info'))
+    insert_execute_command_mock(('borg', 'check', '--info', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -156,7 +212,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_logging_mock(logging.DEBUG)
-    insert_execute_command_mock(('borg', 'check', 'repo', '--debug', '--show-rc'))
+    insert_execute_command_mock(('borg', 'check', '--debug', '--show-rc', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -179,7 +235,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
     insert_execute_command_mock(('borg1', 'check', 'repo'))
 
@@ -197,9 +253,9 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
-    insert_execute_command_mock(('borg', 'check', 'repo', '--remote-path', 'borg1'))
+    insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo'))
 
     module.check_archives(
         repository='repo',
@@ -215,9 +271,9 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
-    insert_execute_command_mock(('borg', 'check', 'repo', '--lock-wait', '5'))
+    insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={'lock_wait': 5}, consistency_config=consistency_config

tests/unit/borg/test_create.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import create as module
@@ -156,7 +157,7 @@ def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
 
 
 DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
+ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 
 def test_create_archive_calls_borg_with_parameters():
@@ -166,7 +167,7 @@ def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND, output_log_level=logging.INFO
+        ('borg', 'create') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -191,7 +192,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
     flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + pattern_flags, output_log_level=logging.INFO
+        ('borg', 'create') + pattern_flags + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -216,7 +217,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + exclude_flags, output_log_level=logging.INFO
+        ('borg', 'create') + exclude_flags + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -239,7 +240,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--stats'),
+        ('borg', 'create', '--list', '--filter', 'AME-', '--info', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.INFO)
@@ -264,7 +265,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
     )
     insert_logging_mock(logging.INFO)
 
@@ -288,7 +289,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc'),
+        ('borg', 'create', '--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc')
+        + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.DEBUG)
@@ -312,7 +314,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -337,7 +339,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--dry-run',), output_log_level=logging.INFO
+        ('borg', 'create', '--dry-run') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -362,7 +364,8 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--dry-run'),
+        ('borg', 'create', '--list', '--filter', 'AME-', '--info', '--dry-run')
+        + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.INFO)
@@ -389,7 +392,8 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run'),
+        ('borg', 'create', '--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run')
+        + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.DEBUG)
@@ -413,7 +417,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--checkpoint-interval', '600'), output_log_level=logging.INFO
+        ('borg', 'create', '--checkpoint-interval', '600') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -435,7 +440,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--chunker-params', '1,2,3,4'), output_log_level=logging.INFO
+        ('borg', 'create', '--chunker-params', '1,2,3,4') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -457,7 +463,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--compression', 'rle'), output_log_level=logging.INFO
+        ('borg', 'create', '--compression', 'rle') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -479,7 +486,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--remote-ratelimit', '100'), output_log_level=logging.INFO
+        ('borg', 'create', '--remote-ratelimit', '100') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -501,7 +509,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--one-file-system',), output_log_level=logging.INFO
+        ('borg', 'create', '--one-file-system') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -524,7 +532,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--numeric-owner',), output_log_level=logging.INFO
+        ('borg', 'create', '--numeric-owner') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -547,7 +555,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--read-special',), output_log_level=logging.INFO
+        ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -563,14 +571,15 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     )
 
 
-def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter():
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND, output_log_level=logging.INFO
+        ('borg', 'create') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -579,21 +588,23 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': True,
+            option_name: True,
             'exclude_patterns': None,
         },
         storage_config={},
     )
 
 
-def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter():
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--nobsdflags',), output_log_level=logging.INFO
+        ('borg', 'create', '--no' + option_name.replace('_', '')) + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -602,7 +613,7 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': False,
+            option_name: False,
             'exclude_patterns': None,
         },
         storage_config={},
@@ -616,7 +627,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--files-cache', 'ctime,size'), output_log_level=logging.INFO
+        ('borg', 'create', '--files-cache', 'ctime,size') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -639,7 +651,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1',) + CREATE_COMMAND[1:], output_log_level=logging.INFO
+        ('borg1', 'create') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -662,7 +674,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.INFO
+        ('borg', 'create', '--remote-path', 'borg1') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -685,7 +698,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--umask', '740'), output_log_level=logging.INFO
+        ('borg', 'create', '--umask', '740') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -707,7 +720,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--lock-wait', '5'), output_log_level=logging.INFO
+        ('borg', 'create', '--lock-wait', '5') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -729,7 +742,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--stats',), output_log_level=logging.WARNING
+        ('borg', 'create', '--stats') + ARCHIVE_WITH_PATHS, output_log_level=logging.WARNING
     )
 
     module.create_archive(
@@ -745,6 +758,29 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter():
     )
 
 
+def test_create_archive_with_progress_calls_borg_with_progress_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command_without_capture').with_args(
+        ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        progress=True,
+    )
+
+
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
@@ -752,7 +788,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -777,7 +813,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
     ).and_return('[]')
 
     json_output = module.create_archive(

tests/unit/borg/test_environment.py

@@ -36,13 +36,27 @@ def test_initialize_with_ssh_command_should_set_environment():
         os.environ = orig_environ
 
 
-def test_initialize_without_configuration_should_not_set_environment():
+def test_initialize_without_configuration_should_only_set_default_environment():
     orig_environ = os.environ
 
     try:
         os.environ = {}
         module.initialize({})
 
-        assert sum(1 for key in os.environ.keys() if key.startswith('BORG_')) == 0
+        assert {key: value for key, value in os.environ.items() if key.startswith('BORG_')} == {
+            'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
+            'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
+        }
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_with_relocated_repo_access_should_override_default():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'relocated_repo_access_is_ok': True})
+        assert os.environ.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
     finally:
         os.environ = orig_environ

tests/unit/borg/test_extract.py

@@ -34,9 +34,9 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise():
 
 def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_parameter():
     insert_execute_command_output_mock(
-        ('borg', 'list', '--short', 'repo', '--info'), result='archive1\narchive2\n'
+        ('borg', 'list', '--short', '--info', 'repo'), result='archive1\narchive2\n'
     )
-    insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive2', '--info'))
+    insert_execute_command_mock(('borg', 'extract', '--dry-run', '--info', 'repo::archive2'))
     insert_logging_mock(logging.INFO)
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
@@ -44,10 +44,10 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet
 
 def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_parameter():
     insert_execute_command_output_mock(
-        ('borg', 'list', '--short', 'repo', '--debug', '--show-rc'), result='archive1\narchive2\n'
+        ('borg', 'list', '--short', '--debug', '--show-rc', 'repo'), result='archive1\narchive2\n'
     )
     insert_execute_command_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--show-rc', '--list')
+        ('borg', 'extract', '--dry-run', '--debug', '--show-rc', '--list', 'repo::archive2')
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -65,10 +65,10 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path():
 
 def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters():
     insert_execute_command_output_mock(
-        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'), result='archive1\narchive2\n'
+        ('borg', 'list', '--short', '--remote-path', 'borg1', 'repo'), result='archive1\narchive2\n'
     )
     insert_execute_command_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1')
+        ('borg', 'extract', '--dry-run', '--remote-path', 'borg1', 'repo::archive2')
     )
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None, remote_path='borg1')
@@ -76,10 +76,10 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters():
 
 def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters():
     insert_execute_command_output_mock(
-        ('borg', 'list', '--short', 'repo', '--lock-wait', '5'), result='archive1\narchive2\n'
+        ('borg', 'list', '--short', '--lock-wait', '5', 'repo'), result='archive1\narchive2\n'
     )
     insert_execute_command_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5')
+        ('borg', 'extract', '--dry-run', '--lock-wait', '5', 'repo::archive2')
     )
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=5)
@@ -99,7 +99,7 @@ def test_extract_archive_calls_borg_with_restore_path_parameters():
 
 
 def test_extract_archive_calls_borg_with_remote_path_parameters():
-    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--remote-path', 'borg1'))
+    insert_execute_command_mock(('borg', 'extract', '--remote-path', 'borg1', 'repo::archive'))
 
     module.extract_archive(
         dry_run=False,
@@ -113,7 +113,7 @@ def test_extract_archive_calls_borg_with_remote_path_parameters():
 
 
 def test_extract_archive_calls_borg_with_numeric_owner_parameter():
-    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--numeric-owner'))
+    insert_execute_command_mock(('borg', 'extract', '--numeric-owner', 'repo::archive'))
 
     module.extract_archive(
         dry_run=False,
@@ -126,7 +126,7 @@ def test_extract_archive_calls_borg_with_numeric_owner_parameter():
 
 
 def test_extract_archive_calls_borg_with_umask_parameters():
-    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--umask', '0770'))
+    insert_execute_command_mock(('borg', 'extract', '--umask', '0770', 'repo::archive'))
 
     module.extract_archive(
         dry_run=False,
@@ -139,7 +139,7 @@ def test_extract_archive_calls_borg_with_umask_parameters():
 
 
 def test_extract_archive_calls_borg_with_lock_wait_parameters():
-    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--lock-wait', '5'))
+    insert_execute_command_mock(('borg', 'extract', '--lock-wait', '5', 'repo::archive'))
 
     module.extract_archive(
         dry_run=False,
@@ -152,7 +152,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters():
 
 
 def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
-    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--info'))
+    insert_execute_command_mock(('borg', 'extract', '--info', 'repo::archive'))
     insert_logging_mock(logging.INFO)
 
     module.extract_archive(
@@ -167,7 +167,7 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
 
 def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
     insert_execute_command_mock(
-        ('borg', 'extract', 'repo::archive', '--debug', '--list', '--show-rc')
+        ('borg', 'extract', '--debug', '--list', '--show-rc', 'repo::archive')
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -182,7 +182,7 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
 
 
 def test_extract_archive_calls_borg_with_dry_run_parameter():
-    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--dry-run'))
+    insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive'))
 
     module.extract_archive(
         dry_run=True,
@@ -195,7 +195,9 @@ def test_extract_archive_calls_borg_with_dry_run_parameter():
 
 
 def test_extract_archive_calls_borg_with_progress_parameter():
-    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--progress'))
+    flexmock(module).should_receive('execute_command_without_capture').with_args(
+        ('borg', 'extract', '--progress', 'repo::archive')
+    ).once()
 
     module.extract_archive(
         dry_run=False,

tests/unit/borg/test_flags.py

@@ -0,0 +1,47 @@
+from flexmock import flexmock
+
+from borgmatic.borg import flags as module
+
+
+def test_make_flags_formats_string_value():
+    assert module.make_flags('foo', 'bar') == ('--foo', 'bar')
+
+
+def test_make_flags_formats_integer_value():
+    assert module.make_flags('foo', 3) == ('--foo', '3')
+
+
+def test_make_flags_formats_true_value():
+    assert module.make_flags('foo', True) == ('--foo',)
+
+
+def test_make_flags_omits_false_value():
+    assert module.make_flags('foo', False) == ()
+
+
+def test_make_flags_formats_name_with_underscore():
+    assert module.make_flags('posix_me_harder', 'okay') == ('--posix-me-harder', 'okay')
+
+
+def test_make_flags_from_arguments_flattens_and_sorts_multiple_arguments():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('make_flags').with_args('baz', 'quux').and_return(
+        ('baz', 'quux')
+    )
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('baz', 'quux', 'foo', 'bar')
+
+
+def test_make_flags_from_arguments_excludes_underscored_argument_names():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', _baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('foo', 'bar')
+
+
+def test_make_flags_from_arguments_omits_excludes():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments, excludes=('baz', 'other')) == ('foo', 'bar')

tests/unit/borg/test_info.py

@@ -1,91 +1,140 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import info as module
 
 from ..test_verbosity import insert_logging_mock
 
-INFO_COMMAND = ('borg', 'info', 'repo')
-
 
 def test_display_archives_info_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND, output_log_level=logging.WARNING
+        ('borg', 'info', 'repo'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config={})
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--info',), output_log_level=logging.WARNING
+        ('borg', 'info', '--info', 'repo'), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.INFO)
-    module.display_archives_info(repository='repo', storage_config={})
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'info', '--json', 'repo'), output_log_level=None
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
-    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'
 
 
 def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--debug', '--show-rc'), output_log_level=logging.WARNING
+        ('borg', 'info', '--debug', '--show-rc', 'repo'), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.display_archives_info(repository='repo', storage_config={})
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'info', '--json', 'repo'), output_log_level=None
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
-    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'
 
 
 def test_display_archives_info_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'info', '--json', 'repo'), output_log_level=None
     ).and_return('[]')
 
-    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'
 
 
-def test_display_archives_info_with_local_path_calls_borg_via_local_path():
+def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1',) + INFO_COMMAND[1:], output_log_level=logging.WARNING
+        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config={}, local_path='borg1')
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive='archive', json=False)
+    )
+
+
+def test_display_archives_info_with_local_path_calls_borg_via_local_path():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING
+    )
+
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False),
+        local_path='borg1',
+    )
 
 
 def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
+        ('borg', 'info', '--remote-path', 'borg1', 'repo'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config={}, remote_path='borg1')
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False),
+        remote_path='borg1',
+    )
 
 
 def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
+        ('borg', 'info', '--lock-wait', '5', 'repo'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config=storage_config)
+    module.display_archives_info(
+        repository='repo',
+        storage_config=storage_config,
+        info_arguments=flexmock(archive=None, json=False),
+    )
+
+
+@pytest.mark.parametrize('argument_name', ('prefix', 'glob_archives', 'sort_by', 'first', 'last'))
+def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'info', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
+        output_log_level=logging.WARNING,
+    )
+
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}),
+    )

tests/unit/borg/test_init.py

@@ -9,7 +9,7 @@ from borgmatic.borg import init as module
 from ..test_verbosity import insert_logging_mock
 
 INFO_SOME_UNKNOWN_EXIT_CODE = -999
-INIT_COMMAND = ('borg', 'init', 'repo', '--encryption', 'repokey')
+INIT_COMMAND = ('borg', 'init', '--encryption', 'repokey')
 
 
 def insert_info_command_found_mock():
@@ -23,21 +23,31 @@ def insert_info_command_not_found_mock():
 
 
 def insert_init_command_mock(init_command, **kwargs):
-    flexmock(module.subprocess).should_receive('check_call').with_args(
-        init_command, **kwargs
+    flexmock(module).should_receive('execute_command_without_capture').with_args(
+        init_command
     ).once()
 
 
 def test_initialize_repository_calls_borg_with_parameters():
     insert_info_command_not_found_mock()
-    insert_init_command_mock(INIT_COMMAND)
+    insert_init_command_mock(INIT_COMMAND + ('repo',))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey')
 
 
+def test_initialize_repository_raises_for_borg_init_error():
+    insert_info_command_not_found_mock()
+    flexmock(module).should_receive('execute_command_without_capture').and_raise(
+        module.subprocess.CalledProcessError(2, 'borg init')
+    )
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.initialize_repository(repository='repo', encryption_mode='repokey')
+
+
 def test_initialize_repository_skips_initialization_when_repository_already_exists():
     insert_info_command_found_mock()
-    flexmock(module.subprocess).should_receive('check_call').never()
+    flexmock(module).should_receive('execute_command_without_capture').never()
 
     module.initialize_repository(repository='repo', encryption_mode='repokey')
 
@@ -53,21 +63,21 @@ def test_initialize_repository_raises_for_unknown_info_command_error():
 
 def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter():
     insert_info_command_not_found_mock()
-    insert_init_command_mock(INIT_COMMAND + ('--append-only',))
+    insert_init_command_mock(INIT_COMMAND + ('--append-only', 'repo'))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', append_only=True)
 
 
 def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter():
     insert_info_command_not_found_mock()
-    insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G'))
+    insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G', 'repo'))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', storage_quota='5G')
 
 
 def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
     insert_info_command_not_found_mock()
-    insert_init_command_mock(INIT_COMMAND + ('--info',))
+    insert_init_command_mock(INIT_COMMAND + ('--info', 'repo'))
     insert_logging_mock(logging.INFO)
 
     module.initialize_repository(repository='repo', encryption_mode='repokey')
@@ -75,7 +85,7 @@ def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
 
 def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
     insert_info_command_not_found_mock()
-    insert_init_command_mock(INIT_COMMAND + ('--debug',))
+    insert_init_command_mock(INIT_COMMAND + ('--debug', 'repo'))
     insert_logging_mock(logging.DEBUG)
 
     module.initialize_repository(repository='repo', encryption_mode='repokey')
@@ -83,13 +93,13 @@ def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_initialize_repository_with_local_path_calls_borg_via_local_path():
     insert_info_command_not_found_mock()
-    insert_init_command_mock(('borg1',) + INIT_COMMAND[1:])
+    insert_init_command_mock(('borg1',) + INIT_COMMAND[1:] + ('repo',))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', local_path='borg1')
 
 
 def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter():
     insert_info_command_not_found_mock()
-    insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1'))
+    insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1', 'repo'))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', remote_path='borg1')

tests/unit/borg/test_list.py

@@ -1,65 +1,88 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import list as module
 
 from ..test_verbosity import insert_logging_mock
 
-LIST_COMMAND = ('borg', 'list', 'repo')
-
 
 def test_list_archives_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND, output_log_level=logging.WARNING
+        ('borg', 'list', 'repo'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, successful=False),
+    )
 
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--info',), output_log_level=logging.WARNING
+        ('borg', 'list', '--info', 'repo'), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, successful=False),
+    )
 
 
 def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'list', '--json', 'repo'), output_log_level=None
     )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=True, successful=False),
+    )
 
 
 def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--debug', '--show-rc'), output_log_level=logging.WARNING
+        ('borg', 'list', '--debug', '--show-rc', 'repo'), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, successful=False),
+    )
 
 
 def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'list', '--json', 'repo'), output_log_level=None
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=True, successful=False),
+    )
 
 
 def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
+        ('borg', 'list', '--lock-wait', '5', 'repo'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config)
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive=None, json=False, successful=False),
+    )
 
 
 def test_list_archives_with_archive_calls_borg_with_archive_parameter():
@@ -68,30 +91,102 @@ def test_list_archives_with_archive_calls_borg_with_archive_parameter():
         ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive='archive', json=False, successful=False),
+    )
 
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1',) + LIST_COMMAND[1:], output_log_level=logging.WARNING
+        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, local_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, successful=False),
+        local_path='borg1',
+    )
 
 
 def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
+        ('borg', 'list', '--remote-path', 'borg1', 'repo'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, successful=False),
+        remote_path='borg1',
+    )
+
+
+def test_list_archives_with_short_calls_borg_with_short_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--short', 'repo'), output_log_level=logging.WARNING
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, successful=False, short=True),
+    )
+
+
+@pytest.mark.parametrize(
+    'argument_name',
+    (
+        'prefix',
+        'glob_archives',
+        'sort_by',
+        'first',
+        'last',
+        'exclude',
+        'exclude_from',
+        'pattern',
+        'pattern_from',
+    ),
+)
+def test_list_archives_passes_through_arguments_to_borg(argument_name):
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
+        output_log_level=logging.WARNING,
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(
+            archive=None, json=False, successful=False, **{argument_name: 'value'}
+        ),
+    )
+
+
+def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--glob-archives', module.BORG_EXCLUDE_CHECKPOINTS_GLOB, 'repo'),
+        output_log_level=logging.WARNING,
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, successful=True),
+    )
 
 
 def test_list_archives_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--json',), output_log_level=None
+        ('borg', 'list', '--json', 'repo'), output_log_level=None
     ).and_return('[]')
 
-    json_output = module.list_archives(repository='repo', storage_config={}, json=True)
+    json_output = module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=True, successful=False),
+    )
 
     assert json_output == '[]'

tests/unit/borg/test_prune.py

@@ -8,8 +8,10 @@ from borgmatic.borg import prune as module
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_execute_command_mock(prune_command, **kwargs):
-    flexmock(module).should_receive('execute_command').with_args(prune_command).once()
+def insert_execute_command_mock(prune_command, output_log_level):
+    flexmock(module).should_receive('execute_command').with_args(
+        prune_command, output_log_level=output_log_level
+    ).once()
 
 
 BASE_PRUNE_FLAGS = (('--keep-daily', '1'), ('--keep-weekly', '2'), ('--keep-monthly', '3'))
@@ -33,17 +35,27 @@ def test_make_prune_flags_accepts_prefix_with_placeholders():
     assert tuple(result) == expected
 
 
-PRUNE_COMMAND = (
-    'borg',
-    'prune',
-    'repo',
-    '--keep-daily',
-    '1',
-    '--keep-weekly',
-    '2',
-    '--keep-monthly',
-    '3',
-)
+def test_make_prune_flags_treats_empty_prefix_as_no_prefix():
+    retention_config = OrderedDict((('keep_daily', 1), ('prefix', '')))
+
+    result = module._make_prune_flags(retention_config)
+
+    expected = (('--keep-daily', '1'),)
+
+    assert tuple(result) == expected
+
+
+def test_make_prune_flags_treats_none_prefix_as_no_prefix():
+    retention_config = OrderedDict((('keep_daily', 1), ('prefix', None)))
+
+    result = module._make_prune_flags(retention_config)
+
+    expected = (('--keep-daily', '1'),)
+
+    assert tuple(result) == expected
+
+
+PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3')
 
 
 def test_prune_archives_calls_borg_with_parameters():
@@ -51,7 +63,7 @@ def test_prune_archives_calls_borg_with_parameters():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND)
+    insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.INFO)
 
     module.prune_archives(
         dry_run=False, repository='repo', storage_config={}, retention_config=retention_config
@@ -63,7 +75,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND + ('--stats', '--info'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--stats', '--info', 'repo'), logging.INFO)
     insert_logging_mock(logging.INFO)
 
     module.prune_archives(
@@ -76,7 +88,9 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND + ('--stats', '--debug', '--list', '--show-rc'))
+    insert_execute_command_mock(
+        PRUNE_COMMAND + ('--stats', '--debug', '--list', '--show-rc', 'repo'), logging.INFO
+    )
     insert_logging_mock(logging.DEBUG)
 
     module.prune_archives(
@@ -89,7 +103,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run',))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO)
 
     module.prune_archives(
         repository='repo', storage_config={}, dry_run=True, retention_config=retention_config
@@ -101,7 +115,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:])
+    insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:] + ('repo',), logging.INFO)
 
     module.prune_archives(
         dry_run=False,
@@ -117,7 +131,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO)
 
     module.prune_archives(
         dry_run=False,
@@ -128,13 +142,29 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     )
 
 
+def test_prune_archives_with_stats_calls_borg_with_stats_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS
+    )
+    insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), logging.WARNING)
+
+    module.prune_archives(
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        retention_config=retention_config,
+        stats=True,
+    )
+
+
 def test_prune_archives_with_umask_calls_borg_with_umask_parameters():
     storage_config = {'umask': '077'}
     retention_config = flexmock()
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO)
 
     module.prune_archives(
         dry_run=False,
@@ -150,7 +180,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO)
 
     module.prune_archives(
         dry_run=False,

tests/unit/commands/test_arguments.py

@@ -0,0 +1,150 @@
+from flexmock import flexmock
+
+from borgmatic.commands import arguments as module
+
+
+def test_parse_subparser_arguments_consumes_subparser_arguments_before_subparser_name():
+    action_namespace = flexmock(foo=True)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(('--foo', 'true', 'action'), subparsers)
+
+    assert arguments == {'action': action_namespace}
+
+
+def test_parse_subparser_arguments_consumes_subparser_arguments_after_subparser_name():
+    action_namespace = flexmock(foo=True)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(('action', '--foo', 'true'), subparsers)
+
+    assert arguments == {'action': action_namespace}
+
+
+def test_parse_subparser_arguments_consumes_subparser_arguments_with_alias():
+    action_namespace = flexmock(foo=True)
+    action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, []))
+    subparsers = flexmock(
+        choices={
+            'action': action_subparser,
+            '-a': action_subparser,
+            'other': flexmock(),
+            '-o': flexmock(),
+        }
+    )
+    flexmock(module).SUBPARSER_ALIASES = {'action': ['-a'], 'other': ['-o']}
+
+    arguments = module.parse_subparser_arguments(('-a', '--foo', 'true'), subparsers)
+
+    assert arguments == {'action': action_namespace}
+
+
+def test_parse_subparser_arguments_consumes_multiple_subparser_arguments():
+    action_namespace = flexmock(foo=True)
+    other_namespace = flexmock(bar=3)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--bar', '3'])
+            ),
+            'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(
+        ('action', '--foo', 'true', 'other', '--bar', '3'), subparsers
+    )
+
+    assert arguments == {'action': action_namespace, 'other': other_namespace}
+
+
+def test_parse_subparser_arguments_applies_default_subparsers():
+    prune_namespace = flexmock()
+    create_namespace = flexmock(progress=True)
+    check_namespace = flexmock()
+    subparsers = flexmock(
+        choices={
+            'prune': flexmock(parse_known_args=lambda arguments: (prune_namespace, ['--progress'])),
+            'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])),
+            'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(('--progress'), subparsers)
+
+    assert arguments == {
+        'prune': prune_namespace,
+        'create': create_namespace,
+        'check': check_namespace,
+    }
+
+
+def test_parse_global_arguments_with_help_does_not_apply_default_subparsers():
+    global_namespace = flexmock(verbosity='lots')
+    action_namespace = flexmock()
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
+            ),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_global_arguments(
+        ('--verbosity', 'lots', '--help'), top_level_parser, subparsers
+    )
+
+    assert arguments == global_namespace
+
+
+def test_parse_global_arguments_consumes_global_arguments_before_subparser_name():
+    global_namespace = flexmock(verbosity='lots')
+    action_namespace = flexmock()
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
+            ),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_global_arguments(
+        ('--verbosity', 'lots', 'action'), top_level_parser, subparsers
+    )
+
+    assert arguments == global_namespace
+
+
+def test_parse_global_arguments_consumes_global_arguments_after_subparser_name():
+    global_namespace = flexmock(verbosity='lots')
+    action_namespace = flexmock()
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
+            ),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_global_arguments(
+        ('action', '--verbosity', 'lots'), top_level_parser, subparsers
+    )
+
+    assert arguments == global_namespace

tests/unit/commands/test_borgmatic.py

@@ -1,101 +1,288 @@
+import logging
+import subprocess
+
 from flexmock import flexmock
 
 from borgmatic.commands import borgmatic as module
 
 
+def test_run_configuration_runs_actions_for_each_repository():
+    flexmock(module.borg_environment).should_receive('initialize')
+    expected_results = [flexmock(), flexmock()]
+    flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return(
+        expected_results[1:]
+    )
+    config = {'location': {'repositories': ['foo', 'bar']}}
+    arguments = {'global': flexmock()}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == expected_results
+
+
+def test_run_configuration_executes_hooks_for_create_action():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.hook).should_receive('execute_hook').twice()
+    flexmock(module).should_receive('run_actions').and_return([])
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
+
+    list(module.run_configuration('test.yaml', config, arguments))
+
+
+def test_run_configuration_logs_actions_error():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.hook).should_receive('execute_hook')
+    expected_results = [flexmock()]
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
+    flexmock(module).should_receive('run_actions').and_raise(OSError)
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(dry_run=False)}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == expected_results
+
+
+def test_run_configuration_logs_pre_hook_error():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.hook).should_receive('execute_hook').and_raise(OSError).and_return(None)
+    expected_results = [flexmock()]
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
+    flexmock(module).should_receive('run_actions').never()
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == expected_results
+
+
+def test_run_configuration_logs_post_hook_error():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(
+        OSError
+    ).and_return(None)
+    expected_results = [flexmock()]
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
+    flexmock(module).should_receive('run_actions').and_return([])
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == expected_results
+
+
+def test_run_configuration_logs_on_error_hook_error():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.hook).should_receive('execute_hook').and_raise(OSError)
+    expected_results = [flexmock(), flexmock()]
+    flexmock(module).should_receive('make_error_log_records').and_return(
+        expected_results[:1]
+    ).and_return(expected_results[1:])
+    flexmock(module).should_receive('run_actions').and_raise(OSError)
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(dry_run=False)}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == expected_results
+
+
+def test_load_configurations_collects_parsed_configurations():
+    configuration = flexmock()
+    other_configuration = flexmock()
+    flexmock(module.validate).should_receive('parse_configuration').and_return(
+        configuration
+    ).and_return(other_configuration)
+
+    configs, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml')))
+
+    assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration}
+    assert logs == []
+
+
+def test_load_configurations_logs_critical_for_parse_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
+
+    configs, logs = tuple(module.load_configurations(('test.yaml',)))
+
+    assert configs == {}
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_make_error_log_records_generates_output_logs_for_message_only():
+    logs = tuple(module.make_error_log_records('Error'))
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_make_error_log_records_generates_output_logs_for_called_process_error():
+    logs = tuple(
+        module.make_error_log_records(
+            'Error', subprocess.CalledProcessError(1, 'ls', 'error output')
+        )
+    )
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+    assert any(log for log in logs if 'error output' in str(log))
+
+
+def test_make_error_log_records_generates_logs_for_value_error():
+    logs = tuple(module.make_error_log_records('Error', ValueError()))
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_make_error_log_records_generates_logs_for_os_error():
+    logs = tuple(module.make_error_log_records('Error', OSError()))
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
+def test_make_error_log_records_generates_nothing_for_other_error():
+    logs = tuple(module.make_error_log_records('Error', KeyError()))
+
+    assert logs == ()
+
+
 def test_collect_configuration_run_summary_logs_info_for_success():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module.hook).should_receive('execute_hook').never()
     flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(extract=False, list=False)
+    arguments = {}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
-    assert all(log for log in logs if log.levelno == module.logging.INFO)
+    assert {log.levelno for log in logs} == {logging.INFO}
+
+
+def test_collect_configuration_run_summary_executes_hooks_for_create():
+    flexmock(module).should_receive('run_configuration').and_return([])
+    arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert {log.levelno for log in logs} == {logging.INFO}
 
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
     flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(extract=True, list=False, repository='repo')
+    arguments = {'extract': flexmock(repository='repo')}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
-    assert all(log for log in logs if log.levelno == module.logging.INFO)
+    assert {log.levelno for log in logs} == {logging.INFO}
 
 
-def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+def test_collect_configuration_run_summary_logs_extract_with_repository_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
         ValueError
     )
-    args = flexmock(extract=True, list=False, repository='repo')
+    expected_logs = (flexmock(),)
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    arguments = {'extract': flexmock(repository='repo')}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert logs == expected_logs
 
 
-def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+def test_collect_configuration_run_summary_logs_missing_configs_error():
+    arguments = {'global': flexmock(config_paths=[])}
+    expected_logs = (flexmock(),)
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+
+    logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
+
+    assert logs == expected_logs
+
+
+def test_collect_configuration_run_summary_logs_pre_hook_error():
+    flexmock(module.hook).should_receive('execute_hook').and_raise(ValueError)
+    expected_logs = (flexmock(),)
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert logs == expected_logs
+
+
+def test_collect_configuration_run_summary_logs_post_hook_error():
+    flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(ValueError)
+    flexmock(module).should_receive('run_configuration').and_return([])
+    expected_logs = (flexmock(),)
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert expected_logs[0] in logs
+
+
+def test_collect_configuration_run_summary_logs_for_list_with_archive_and_repository_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
         ValueError
     )
-    args = flexmock(extract=False, list=True, repository='repo', archive='test')
+    expected_logs = (flexmock(),)
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
+    arguments = {'list': flexmock(repository='repo', archive='test')}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert logs == expected_logs
 
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_list():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(extract=False, list=True, repository='repo', archive=None)
+    arguments = {'list': flexmock(repository='repo', archive=None)}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
-    assert all(log for log in logs if log.levelno == module.logging.INFO)
+    assert {log.levelno for log in logs} == {logging.INFO}
 
 
-def test_collect_configuration_run_summary_logs_critical_for_parse_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
-    args = flexmock(extract=False, list=False)
-
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
-
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
-
-
-def test_collect_configuration_run_summary_logs_critical_for_run_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+def test_collect_configuration_run_summary_logs_run_configuration_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module).should_receive('run_configuration').and_raise(ValueError)
-    args = flexmock(extract=False, list=False)
+    flexmock(module).should_receive('run_configuration').and_return(
+        [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))]
+    )
+    arguments = {}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
-
-
-def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
-    flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(config_paths=(), extract=False, list=False)
-
-    logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args))
-
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
 
 
 def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
-    flexmock(module.validate).should_receive('parse_configuration').and_return(
-        {'test.yaml': {}, 'test2.yaml': {}}
-    )
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
         ['baz']
     )
     flexmock(module.sys.stdout).should_receive('write').with_args('["foo", "bar", "baz"]').once()
-    args = flexmock(extract=False, list=False)
+    arguments = {}
 
-    tuple(module.collect_configuration_run_summary_logs(('test.yaml', 'test2.yaml'), args=args))
+    tuple(
+        module.collect_configuration_run_summary_logs(
+            {'test.yaml': {}, 'test2.yaml': {}}, arguments=arguments
+        )
+    )

tests/unit/config/test_validate.py

@@ -1,5 +1,4 @@
 import pytest
-from flexmock import flexmock
 
 from borgmatic.config import validate as module
 
@@ -37,20 +36,6 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
         )
 
 
-def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix():
-    logger = flexmock(module.logger)
-    logger.should_receive('warning').once()
-
-    module.apply_logical_validation(
-        'config.yaml',
-        {
-            'storage': {'archive_name_format': '{hostname}-{now}'},
-            'retention': {'prefix': '{hostname}-'},
-            'consistency': {},
-        },
-    )
-
-
 def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories():
     with pytest.raises(module.Validation_error):
         module.apply_logical_validation(
@@ -74,10 +59,7 @@ def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_re
     )
 
 
-def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
-    logger = flexmock(module.logger)
-    logger.should_receive('warning').never()
-
+def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
     module.apply_logical_validation(
         'config.yaml',
         {

tests/unit/test_execute.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic import execute as module
@@ -49,3 +50,28 @@ def test_execute_command_captures_output_with_shell():
     output = module.execute_command(full_command, output_log_level=None, shell=True)
 
     assert output == expected_output
+
+
+def test_execute_command_without_capture_does_not_raise_on_success():
+    flexmock(module.subprocess).should_receive('check_call').and_raise(
+        module.subprocess.CalledProcessError(0, 'borg init')
+    )
+
+    module.execute_command_without_capture(('borg', 'init'))
+
+
+def test_execute_command_without_capture_does_not_raise_on_warning():
+    flexmock(module.subprocess).should_receive('check_call').and_raise(
+        module.subprocess.CalledProcessError(1, 'borg init')
+    )
+
+    module.execute_command_without_capture(('borg', 'init'))
+
+
+def test_execute_command_without_capture_raises_on_error():
+    flexmock(module.subprocess).should_receive('check_call').and_raise(
+        module.subprocess.CalledProcessError(2, 'borg init')
+    )
+
+    with pytest.raises(module.subprocess.CalledProcessError):
+        module.execute_command_without_capture(('borg', 'init'))

tests/unit/test_hook.py

@@ -5,7 +5,24 @@ from flexmock import flexmock
 from borgmatic import hook as module
 
 
+def test_interpolate_context_passes_through_command_without_variable():
+    assert module.interpolate_context('ls', {'foo': 'bar'}) == 'ls'
+
+
+def test_interpolate_context_passes_through_command_with_unknown_variable():
+    assert module.interpolate_context('ls {baz}', {'foo': 'bar'}) == 'ls {baz}'
+
+
+def test_interpolate_context_interpolates_variables():
+    context = {'foo': 'bar', 'baz': 'quux'}
+
+    assert module.interpolate_context('ls {foo}{baz} {baz}', context) == 'ls barquux quux'
+
+
 def test_execute_hook_invokes_each_command():
+    flexmock(module).should_receive('interpolate_context').replace_with(
+        lambda command, context: command
+    )
     flexmock(module.execute).should_receive('execute_command').with_args(
         [':'], output_log_level=logging.WARNING, shell=True
     ).once()
@@ -14,6 +31,9 @@ def test_execute_hook_invokes_each_command():
 
 
 def test_execute_hook_with_multiple_commands_invokes_each_command():
+    flexmock(module).should_receive('interpolate_context').replace_with(
+        lambda command, context: command
+    )
     flexmock(module.execute).should_receive('execute_command').with_args(
         [':'], output_log_level=logging.WARNING, shell=True
     ).once()
@@ -25,6 +45,9 @@ def test_execute_hook_with_multiple_commands_invokes_each_command():
 
 
 def test_execute_hook_with_umask_sets_that_umask():
+    flexmock(module).should_receive('interpolate_context').replace_with(
+        lambda command, context: command
+    )
     flexmock(module.os).should_receive('umask').with_args(0o77).and_return(0o22).once()
     flexmock(module.os).should_receive('umask').with_args(0o22).once()
     flexmock(module.execute).should_receive('execute_command').with_args(
@@ -35,6 +58,9 @@ def test_execute_hook_with_umask_sets_that_umask():
 
 
 def test_execute_hook_with_dry_run_skips_commands():
+    flexmock(module).should_receive('interpolate_context').replace_with(
+        lambda command, context: command
+    )
     flexmock(module.execute).should_receive('execute_command').never()
 
     module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=True)
@@ -45,8 +71,41 @@ def test_execute_hook_with_empty_commands_does_not_raise():
 
 
 def test_execute_hook_on_error_logs_as_error():
+    flexmock(module).should_receive('interpolate_context').replace_with(
+        lambda command, context: command
+    )
     flexmock(module.execute).should_receive('execute_command').with_args(
         [':'], output_log_level=logging.ERROR, shell=True
     ).once()
 
     module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
+
+
+def test_ping_healthchecks_hits_ping_url():
+    ping_url = 'https://example.com'
+    flexmock(module.requests).should_receive('get').with_args(ping_url)
+
+    module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False)
+
+
+def test_ping_healthchecks_without_ping_url_does_not_raise():
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_healthchecks(ping_url_or_uuid=None, config_filename='config.yaml', dry_run=False)
+
+
+def test_ping_healthchecks_with_ping_uuid_hits_corresponding_url():
+    ping_uuid = 'abcd-efgh-ijkl-mnop'
+    flexmock(module.requests).should_receive('get').with_args(
+        'https://hc-ping.com/{}'.format(ping_uuid)
+    )
+
+    module.ping_healthchecks(ping_uuid, 'config.yaml', dry_run=False)
+
+
+def test_ping_healthchecks_hits_ping_url_with_append():
+    ping_url = 'https://example.com'
+    append = 'failed-so-hard'
+    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
+
+    module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append)

tests/unit/test_logger.py

@@ -20,52 +20,107 @@ def test_to_bool_passes_none_through():
     assert module.to_bool(None) is None
 
 
+def test_interactive_console_false_when_not_isatty(capsys):
+    with capsys.disabled():
+        flexmock(module.sys.stdout).should_receive('isatty').and_return(False)
+
+        assert module.interactive_console() is False
+
+
+def test_interactive_console_false_when_TERM_is_dumb(capsys):
+    with capsys.disabled():
+        flexmock(module.sys.stdout).should_receive('isatty').and_return(True)
+        flexmock(module.os.environ).should_receive('get').with_args('TERM').and_return('dumb')
+
+        assert module.interactive_console() is False
+
+
+def test_interactive_console_true_when_isatty_and_TERM_is_not_dumb(capsys):
+    with capsys.disabled():
+        flexmock(module.sys.stdout).should_receive('isatty').and_return(True)
+        flexmock(module.os.environ).should_receive('get').with_args('TERM').and_return('smart')
+
+        assert module.interactive_console() is True
+
+
 def test_should_do_markup_respects_no_color_value():
-    assert module.should_do_markup(no_color=True) is False
+    assert module.should_do_markup(no_color=True, configs={}) is False
+
+
+def test_should_do_markup_respects_config_value():
+    assert (
+        module.should_do_markup(no_color=False, configs={'foo.yaml': {'output': {'color': False}}})
+        is False
+    )
+
+
+def test_should_do_markup_prefers_any_false_config_value():
+    assert (
+        module.should_do_markup(
+            no_color=False,
+            configs={
+                'foo.yaml': {'output': {'color': True}},
+                'bar.yaml': {'output': {'color': False}},
+            },
+        )
+        is False
+    )
 
 
 def test_should_do_markup_respects_PY_COLORS_environment_variable():
     flexmock(module.os.environ).should_receive('get').and_return('True')
     flexmock(module).should_receive('to_bool').and_return(True)
 
-    assert module.should_do_markup(no_color=False) is True
+    assert module.should_do_markup(no_color=False, configs={}) is True
+
+
+def test_should_do_markup_prefers_no_color_value_to_config_value():
+    assert (
+        module.should_do_markup(no_color=True, configs={'foo.yaml': {'output': {'color': True}}})
+        is False
+    )
+
+
+def test_should_do_markup_prefers_config_value_to_PY_COLORS():
+    flexmock(module.os.environ).should_receive('get').and_return('True')
+    flexmock(module).should_receive('to_bool').and_return(True)
+
+    assert (
+        module.should_do_markup(no_color=False, configs={'foo.yaml': {'output': {'color': False}}})
+        is False
+    )
 
 
 def test_should_do_markup_prefers_no_color_value_to_PY_COLORS():
     flexmock(module.os.environ).should_receive('get').and_return('True')
     flexmock(module).should_receive('to_bool').and_return(True)
 
-    assert module.should_do_markup(no_color=True) is False
+    assert module.should_do_markup(no_color=True, configs={}) is False
 
 
-def test_should_do_markup_respects_stdout_tty_value():
+def test_should_do_markup_respects_interactive_console_value():
     flexmock(module.os.environ).should_receive('get').and_return(None)
+    flexmock(module).should_receive('interactive_console').and_return(True)
 
-    assert module.should_do_markup(no_color=False) is False
+    assert module.should_do_markup(no_color=False, configs={}) is True
 
 
-def test_should_do_markup_prefers_PY_COLORS_to_stdout_tty_value():
+def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value():
     flexmock(module.os.environ).should_receive('get').and_return('True')
     flexmock(module).should_receive('to_bool').and_return(True)
+    flexmock(module).should_receive('interactive_console').and_return(False)
 
-    assert module.should_do_markup(no_color=False) is True
+    assert module.should_do_markup(no_color=False, configs={}) is True
 
 
-@pytest.mark.parametrize('method_name', ('critical', 'error', 'warn', 'info', 'debug'))
-def test_borgmatic_logger_log_method_does_not_raise(method_name):
-    flexmock(module).should_receive('color_text')
-    flexmock(module.logging.Logger).should_receive(method_name)
+def test_console_color_formatter_format_includes_log_message():
+    plain_message = 'uh oh'
+    record = flexmock(levelno=logging.CRITICAL, msg=plain_message)
 
-    getattr(module.Borgmatic_logger('test'), method_name)(msg='hi')
+    colored_message = module.Console_color_formatter().format(record)
 
-
-def test_borgmatic_logger_handle_does_not_raise():
-    flexmock(module).should_receive('color_text')
-    flexmock(module.logging.Logger).should_receive('handle')
-
-    module.Borgmatic_logger('test').handle(
-        module.logging.makeLogRecord(dict(levelno=module.logging.CRITICAL, msg='hi'))
-    )
+    assert colored_message != plain_message
+    assert plain_message in colored_message
 
 
 def test_color_text_does_not_raise():
@@ -77,11 +132,12 @@ def test_color_text_without_color_does_not_raise():
 
 
 def test_configure_logging_probes_for_log_socket_on_linux():
+    flexmock(module).should_receive('Console_color_formatter')
+    flexmock(module).should_receive('interactive_console').and_return(False)
     flexmock(module.logging).should_receive('basicConfig').with_args(
         level=logging.INFO, handlers=tuple
     )
     flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True)
-    flexmock(module.os.path).should_receive('exists').with_args('/var/run/syslog').and_return(False)
     syslog_handler = logging.handlers.SysLogHandler()
     flexmock(module.logging.handlers).should_receive('SysLogHandler').with_args(
         address='/dev/log'
@@ -91,6 +147,8 @@ def test_configure_logging_probes_for_log_socket_on_linux():
 
 
 def test_configure_logging_probes_for_log_socket_on_macos():
+    flexmock(module).should_receive('Console_color_formatter')
+    flexmock(module).should_receive('interactive_console').and_return(False)
     flexmock(module.logging).should_receive('basicConfig').with_args(
         level=logging.INFO, handlers=tuple
     )
@@ -105,6 +163,7 @@ def test_configure_logging_probes_for_log_socket_on_macos():
 
 
 def test_configure_logging_sets_global_logger_to_most_verbose_log_level():
+    flexmock(module).should_receive('Console_color_formatter')
     flexmock(module.logging).should_receive('basicConfig').with_args(
         level=logging.DEBUG, handlers=tuple
     ).once()
@@ -114,6 +173,7 @@ def test_configure_logging_sets_global_logger_to_most_verbose_log_level():
 
 
 def test_configure_logging_skips_syslog_if_not_found():
+    flexmock(module).should_receive('Console_color_formatter')
     flexmock(module.logging).should_receive('basicConfig').with_args(
         level=logging.INFO, handlers=tuple
     )
@@ -121,3 +181,15 @@ def test_configure_logging_skips_syslog_if_not_found():
     flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
 
     module.configure_logging(console_log_level=logging.INFO)
+
+
+def test_configure_logging_skips_syslog_if_interactive_console():
+    flexmock(module).should_receive('Console_color_formatter')
+    flexmock(module).should_receive('interactive_console').and_return(True)
+    flexmock(module.logging).should_receive('basicConfig').with_args(
+        level=logging.INFO, handlers=tuple
+    )
+    flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True)
+    flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
+
+    module.configure_logging(console_log_level=logging.INFO)