Only log to syslog when run from a non-interactive console (e.g. a cron job). Related to #197.

Closes https://projects.torsion.org/witten/borgmatic/issues/178.

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.tox

.drone.yml

@@ -1,18 +1,57 @@
-pipeline:
-  build:
-    image: python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
-    pull: true
-    commands:
-      - pip install tox
-      - tox
-      - apk add --no-cache borgbackup
-      - tox -e end-to-end
+---
+kind: pipeline
+name: python-3-5-alpine-3-9
 
-matrix:
-  ALPINE_VERSION:
-    - 3.7
-    - 3.8
-  PYTHON_VERSION:
-    - 3.5
-    - 3.6
-    - 3.7
+steps:
+- name: build
+  image: python:3.5-alpine3.9
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-6-alpine-3-9
+
+steps:
+- name: build
+  image: python:3.6-alpine3.9
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-9
+
+steps:
+- name: build
+  image: python:3.7-alpine3.9
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-7
+
+steps:
+- name: build
+  image: python:3.7-alpine3.7
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: documentation
+
+steps:
+- name: build
+  image: plugins/docker
+  settings:
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    repo: witten/borgmatic-docs
+    dockerfile: docs/Dockerfile
+  when:
+    branch:
+      - master

.eleventy.js

@@ -0,0 +1,41 @@
+const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
+const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
+
+module.exports = function(eleventyConfig) {
+    eleventyConfig.addPlugin(pluginSyntaxHighlight);
+    eleventyConfig.addPlugin(inclusiveLangPlugin);
+
+    let markdownIt = require("markdown-it");
+    let markdownItAnchor = require("markdown-it-anchor");
+    let markdownItReplaceLink = require("markdown-it-replace-link");
+
+    let markdownItOptions = {
+        html: true,
+        breaks: false,
+        linkify: true,
+        // Replace links to .md files with links to directories. This allows unparsed Markdown links
+        // to work on GitHub, while rendered links elsewhere also work.
+        replaceLink: function (link, env) {
+            return link.replace(/\.md$/, '/');
+        }
+    };
+    let markdownItAnchorOptions = {
+        permalink: true,
+        permalinkClass: "direct-link"
+    };
+
+    eleventyConfig.setLibrary(
+        "md",
+        markdownIt(markdownItOptions)
+            .use(markdownItAnchor, markdownItAnchorOptions)
+            .use(markdownItReplaceLink)
+    );
+
+    return {
+        templateFormats: [
+          "md",
+          "txt"
+        ]
+    }
+};
+

.gitea/issue_template.md

@@ -0,0 +1,31 @@
+#### What I'm trying to do and why
+
+#### Steps to reproduce (if a bug)
+
+Include (sanitized) borgmatic configuration files if applicable.
+
+#### Actual behavior (if a bug)
+
+Include (sanitized) `--verbosity 2` output if applicable.
+
+#### Expected behavior (if a bug)
+
+#### Other notes / implementation ideas
+
+#### Environment
+
+**borgmatic version:** [version here]
+
+Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+
+**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
+
+**Borg version:** [version here]
+
+Use `sudo borg --version`
+
+**Python version:** [version here]
+
+Use `python --version`
+
+**operating system and version:** [OS here]

.gitignore

@@ -5,5 +5,6 @@
 .coverage
 .pytest_cache
 .tox
-build
-dist
+build/
+dist/
+pip-wheel-metadata/

AUTHORS

@@ -7,6 +7,6 @@ Johannes Feichtner: Support for user hooks
 Michele Lazzeri: Custom archive names
 Nick Whyte: Support prefix filtering for archive consistency checks
 newtonne: Read encryption password from external file
-Robin `ypid` Schneider: Support additional options of Borg
+Robin `ypid` Schneider: Support additional options of Borg and add validate-borgmatic-config command
 Scott Squires: Custom archive names
 Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option

NEWS

@@ -1,3 +1,63 @@
+1.3.12
+ * Only log to syslog when run from a non-interactive console (e.g. a cron job).
+ * Remove unicode byte order mark from syslog output so it doesn't show up as a literal in rsyslog
+   output. See discussion on #197.
+
+1.3.11
+ * #193: Pass through several "borg list" and "borg info" flags like --short, --format, --sort-by,
+   --first, --last, etc. via borgmatic command-line flags.
+ * Add borgmatic info --repository and --archive command-line flags to display info for individual
+   repositories or archives.
+ * Support for Borg --noatime, --noctime, and --nobirthtime flags via corresponding options in
+   borgmatic configuration location section.
+
+1.3.10
+ * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.
+
+1.3.9
+ * #195: Switch to command-line actions as more traditional sub-commands, e.g. "borgmatic create",
+   "borgmatic prune", etc. However, the classic dashed options like "--create" still work!
+
+1.3.8
+ * #191: Disable console color via "color" option in borgmatic configuration output section.
+
+1.3.7
+ * #196: Fix for unclear error message for invalid YAML merge include.
+ * #197: Don't color syslog output.
+ * Change default syslog verbosity to show errors only.
+
+1.3.6
+ * #53: Log to syslog in addition to existing console logging. Add --syslog-verbosity flag to
+   customize the log level. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/
+ * #178: Look for .yml configuration file extension in addition to .yaml.
+ * #189: Set umask used when executing hooks via "umask" option in borgmatic hooks section.
+ * Remove Python cache files before each Tox run.
+ * Add #borgmatic Freenode IRC channel to documentation.
+ * Add Borg/borgmatic hosting providers section to documentation.
+ * Add files for building documentation into a Docker image for web serving.
+ * Upgrade project build server from Drone 0.8 to 1.1.
+ * Build borgmatic documentation during continuous integration.
+ * We're nearly at 500 ★s on GitHub. We can do this!
+
+1.3.5
+ * #153: Support for various Borg directory environment variables (BORG_CONFIG_DIR, BORG_CACHE_DIR,
+   etc.) via options in borgmatic's storage configuration.
+ * #177: Fix for regression with missing verbose log entries.
+
+1.3.4
+ * Part of #125: Color borgmatic (but not Borg) output when using an interactive terminal.
+ * #166: Run tests for all installed versions of Python.
+ * #168: Update README with continuous integration badge.
+ * #169: Automatically sort Python imports in code.
+ * Document installing borgmatic with pip install --user instead of a system Python install.
+ * Get more reproducible builds by pinning the versions of pip and tox used to run tests.
+ * Factor out build/test configuration from tox.ini file.
+
+1.3.3
+ * Add validate-borgmatic-config command, useful for validating borgmatic config generated by
+   configuration management or even edited by hand.
+
 1.3.2
  * #160: Fix for hooks executing when using --dry-run. Now hooks are skipped during a dry run.
 

README.md

@@ -1,17 +1,20 @@
 ---
 title: borgmatic
-permalink: borgmatic/index.html
+permalink: index.html
 ---
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
+
 ## Overview
 
-<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" width="150px" style="float: right; padding-left: 1em;">
+<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
 
-borgmatic is a simple Python wrapper script for the
-[Borg](https://www.borgbackup.org/) backup software that initiates a backup,
-prunes any old backups according to a retention policy, and validates backups
-for consistency. The script supports specifying your settings in a declarative
-configuration file rather than having to put them all on the command-line, and
-handles common errors.
+borgmatic is simple, configuration-driven backup software for servers and
+workstations. Backup all of your machines from the command-line or scheduled
+jobs. No GUI required. Built atop [Borg Backup](https://www.borgbackup.org/),
+borgmatic initiates a backup, prunes any old backups according to a retention
+policy, and validates backups for consistency. borgmatic supports specifying
+your settings in a declarative configuration file, rather than having to put
+them all on the command-line, and handles common errors.
 
 Here's an example config file:
 
@@ -56,22 +59,34 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 
 ## How-to guides
 
- * [Set up backups with borgmatic](docs/how-to/set-up-backups.md) ⬅ *Start here!*
- * [Make per-application backups](docs/how-to/make-per-application-backups.md)
- * [Deal with very large backups](docs/how-to/deal-with-very-large-backups.md)
- * [Inspect your backups](docs/how-to/inspect-your-backups.md)
- * [Restore a backup](docs/how-to/restore-a-backup.md)
- * [Run preparation steps before backups](docs/how-to/run-preparation-steps-before-backups.md)
- * [Upgrade borgmatic](docs/how-to/upgrade.md)
- * [Develop on borgmatic](docs/how-to/develop-on-borgmatic.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) ⬅ *Start here!*
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
+ * [Run preparation steps before backups](https://torsion.org/borgmatic/docs/how-to/run-preparation-steps-before-backups/)
+ * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 
 
 ## Reference guides
 
- * [borgmatic configuration reference](docs/reference/configuration.md)
- * [borgmatic command-line reference](docs/reference/command-line.md)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)
 
 
+## Hosting providers
+
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
+
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
+
 ## Support and contributing
 
 ### Issues
@@ -82,6 +97,11 @@ create a new issue or comment on an issue, you'll need to [login
 first](https://projects.torsion.org/user/login). Note that you can login with
 an existing GitHub account if you prefer.
 
+If you'd like to chat with borgmatic developers or users, head on over to the
+`#borgmatic` IRC channel on Freenode, either via <a
+href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
+native <a href="irc://chat.freenode.net:6697">IRC client</a>.
+
 Other questions or comments? Contact <mailto:witten@torsion.org>.
 
 
@@ -94,5 +114,10 @@ to discuss your idea. We also accept Pull Requests on GitHub, if that's more
 your thing. In general, contributions are very welcome. We don't bite! 
 
 Also, please check out the [borgmatic development
-how-to](docs/how-to/develop-on-borgmatic.md) for info on cloning source code,
-running tests, etc.
+how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
+info on cloning source code, running tests, etc.
+
+<script>
+  var links = document.getElementsByClassName("referral");
+  links[Math.floor(Math.random() * links.length)].style.display = "none";
+</script>

borgmatic/borg/check.py

@@ -1,9 +1,7 @@
 import logging
-import os
-import subprocess
 
 from borgmatic.borg import extract
-
+from borgmatic.execute import execute_command
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
@@ -114,11 +112,7 @@ def check_archives(
             + verbosity_flags
         )
 
-        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-        stdout = None if verbosity_flags else open(os.devnull, 'w')
-
-        logger.debug(' '.join(full_command))
-        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+        execute_command(full_command)
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -4,8 +4,7 @@ import logging
 import os
 import tempfile
 
-from borgmatic.borg.execute import execute_command
-
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -143,19 +142,33 @@ def create_archive(
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--noatime',) if location_config.get('atime') is False else ())
+        + (('--noctime',) if location_config.get('ctime') is False else ())
+        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
         + (('--read-special',) if location_config.get('read_special') else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--list', '--filter', 'AME-') if logger.isEnabledFor(logging.INFO) else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--stats',) if not dry_run and (logger.isEnabledFor(logging.INFO) or stats) else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--list', '--filter', 'AME-') if logger.isEnabledFor(logging.INFO) and not json else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (
+            ('--stats',)
+            if not dry_run and (logger.isEnabledFor(logging.INFO) or stats) and not json
+            else ()
+        )
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
         + (('--dry-run',) if dry_run else ())
         + (('--progress',) if progress else ())
         + (('--json',) if json else ())
     )
 
-    return execute_command(full_command, capture_output=json)
+    if json:
+        output_log_level = None
+    elif stats:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    return execute_command(full_command, output_log_level)

borgmatic/borg/environment.py

@@ -1,15 +1,19 @@
 import os
 
+OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'borg_base_directory': 'BORG_BASE_DIR',
+    'borg_config_directory': 'BORG_CONFIG_DIR',
+    'borg_cache_directory': 'BORG_CACHE_DIR',
+    'borg_security_directory': 'BORG_SECURITY_DIR',
+    'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
+    'ssh_command': 'BORG_RSH',
+}
+
 
 def initialize(storage_config):
-    passcommand = storage_config.get('encryption_passcommand')
-    if passcommand:
-        os.environ['BORG_PASSCOMMAND'] = passcommand
-
-    passphrase = storage_config.get('encryption_passphrase')
-    if passphrase:
-        os.environ['BORG_PASSPHRASE'] = passphrase
-
-    ssh_command = storage_config.get('ssh_command')
-    if ssh_command:
-        os.environ['BORG_RSH'] = ssh_command
+    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name)
+        if value:
+            os.environ[environment_variable_name] = value

borgmatic/borg/execute.py

@@ -1,19 +0,0 @@
-import logging
-import subprocess
-
-
-logger = logging.getLogger(__name__)
-
-
-def execute_command(full_command, capture_output=False):
-    '''
-    Execute the given command (a sequence of command/argument strings). If capture output is True,
-    then return the command's output as a string.
-    '''
-    logger.debug(' '.join(full_command))
-
-    if capture_output:
-        output = subprocess.check_output(full_command)
-        return output.decode() if output is not None else None
-    else:
-        subprocess.check_call(full_command)

borgmatic/borg/extract.py

@@ -1,7 +1,6 @@
 import logging
-import sys
-import subprocess
 
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -26,10 +25,11 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         + verbosity_flags
     )
 
-    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
+    list_output = execute_command(full_list_command, output_log_level=None)
 
-    last_archive_name = list_output.strip().split('\n')[-1]
-    if not last_archive_name:
+    try:
+        last_archive_name = list_output.strip().splitlines()[-1]
+    except IndexError:
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
@@ -48,8 +48,7 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         + list_flag
     )
 
-    logger.debug(' '.join(full_extract_command))
-    subprocess.check_call(full_extract_command)
+    execute_command(full_extract_command)
 
 
 def extract_archive(
@@ -84,5 +83,4 @@ def extract_archive(
         + (('--progress',) if progress else ())
     )
 
-    logger.debug(' '.join(full_command))
-    subprocess.check_call(full_command)
+    execute_command(full_command)

borgmatic/borg/flags.py

@@ -0,0 +1,31 @@
+import itertools
+
+
+def make_flags(name, value):
+    '''
+    Given a flag name and its value, return it formatted as Borg-compatible flags.
+    '''
+    if not value:
+        return ()
+
+    flag = '--{}'.format(name.replace('_', '-'))
+
+    if value is True:
+        return (flag,)
+
+    return (flag, str(value))
+
+
+def make_flags_from_arguments(arguments, excludes=()):
+    '''
+    Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
+    list of named arguments to exclude, generate and return the corresponding Borg command-line
+    flags as a tuple.
+    '''
+    return tuple(
+        itertools.chain.from_iterable(
+            make_flags(name, value=getattr(arguments, name))
+            for name in sorted(vars(arguments))
+            if name not in excludes and not name.startswith('_')
+        )
+    )

borgmatic/borg/info.py

@@ -1,27 +1,44 @@
 import logging
 
-from borgmatic.borg.execute import execute_command
-
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
 def display_archives_info(
-    repository, storage_config, local_path='borg', remote_path=None, json=False
+    repository, storage_config, info_arguments, local_path='borg', remote_path=None
 ):
     '''
-    Given a local or remote repository path, and a storage config dict, display summary information
-    for Borg archives in the repository or return JSON summary information.
+    Given a local or remote repository path, a storage config dict, and the arguments to the info
+    action, display summary information for Borg archives in the repository or return JSON summary
+    information.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'info', repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--json',) if json else ())
+        (
+            local_path,
+            'info',
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
+        )
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not info_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not info_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
     )
 
-    return execute_command(full_command, capture_output=json)
+    return execute_command(
+        full_command, output_log_level=None if info_arguments.json else logging.WARNING
+    )

borgmatic/borg/init.py

@@ -1,10 +1,14 @@
 import logging
 import subprocess
 
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
+INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
+
+
 def initialize_repository(
     repository,
     encryption_mode,
@@ -21,9 +25,13 @@ def initialize_repository(
     info_command = (local_path, 'info', repository)
     logger.debug(' '.join(info_command))
 
-    if subprocess.call(info_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0:
+    try:
+        execute_command(info_command, output_log_level=None)
         logger.info('Repository already exists. Skipping initialization.')
         return
+    except subprocess.CalledProcessError as error:
+        if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
+            raise
 
     init_command = (
         (local_path, 'init', repository)
@@ -35,5 +43,5 @@ def initialize_repository(
         + (('--remote-path', remote_path) if remote_path else ())
     )
 
-    logger.debug(' '.join(init_command))
+    # Don't use execute_command() here because it doesn't support interactive prompts.
     subprocess.check_call(init_command)

borgmatic/borg/list.py

@@ -1,28 +1,42 @@
 import logging
 
-from borgmatic.borg.execute import execute_command
-
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def list_archives(
-    repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
-):
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
-    Given a local or remote repository path and a storage config dict, display the output of listing
-    Borg archives in the repository or return JSON output. Or, if an archive name is given, listing
-    the files in that archive.
+    Given a local or remote repository path, a storage config dict, and the arguments to the list
+    action, display the output of listing Borg archives in the repository or return JSON output. Or,
+    if an archive name is given, listing the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--json',) if json else ())
+        (
+            local_path,
+            'list',
+            '::'.join((repository, list_arguments.archive))
+            if list_arguments.archive
+            else repository,
+        )
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(list_arguments, excludes=('repository', 'archive'))
     )
 
-    return execute_command(full_command, capture_output=json)
+    return execute_command(
+        full_command, output_log_level=None if list_arguments.json else logging.WARNING
+    )

borgmatic/borg/prune.py

@@ -1,6 +1,6 @@
 import logging
-import subprocess
 
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -60,5 +60,4 @@ def prune_archives(
         + (('--stats',) if stats else ())
     )
 
-    logger.debug(' '.join(full_command))
-    subprocess.check_call(full_command)
+    execute_command(full_command)

borgmatic/commands/arguments.py

@@ -0,0 +1,352 @@
+import collections
+from argparse import ArgumentParser
+
+from borgmatic.config import collect
+
+SUBPARSER_ALIASES = {
+    'init': ['--init', '-I'],
+    'prune': ['--prune', '-p'],
+    'create': ['--create', '-C'],
+    'check': ['--check', '-k'],
+    'extract': ['--extract', '-x'],
+    'list': ['--list', '-l'],
+    'info': ['--info', '-i'],
+}
+
+
+def parse_subparser_arguments(unparsed_arguments, top_level_parser, subparsers):
+    '''
+    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
+    object as returned by argparse.ArgumentParser().add_subparsers(), ask each subparser to parse
+    its own arguments and the top-level parser to parse any remaining arguments.
+
+    Return the result as a dict mapping from subparser name (or "global") to a parsed namespace of
+    arguments.
+    '''
+    arguments = collections.OrderedDict()
+    remaining_arguments = list(unparsed_arguments)
+    alias_to_subparser_name = {
+        alias: subparser_name
+        for subparser_name, aliases in SUBPARSER_ALIASES.items()
+        for alias in aliases
+    }
+
+    # Give each requested action's subparser a shot at parsing all arguments.
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in unparsed_arguments:
+            continue
+
+        remaining_arguments.remove(subparser_name)
+        canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name)
+
+        parsed, remaining = subparser.parse_known_args(unparsed_arguments)
+        arguments[canonical_name] = parsed
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            parsed, remaining = subparser.parse_known_args(unparsed_arguments)
+            arguments[subparser_name] = parsed
+
+    # Then ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
+    # are global arguments.
+    for subparser_name in arguments.keys():
+        subparser = subparsers.choices[subparser_name]
+        parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    arguments['global'] = top_level_parser.parse_args(remaining_arguments)
+
+    return arguments
+
+
+def parse_arguments(*unparsed_arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    global_parser = ArgumentParser(add_help=False)
+    global_group = global_parser.add_argument_group('global arguments')
+
+    global_group.add_argument(
+        '-c',
+        '--config',
+        nargs='*',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+    global_group.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    global_group.add_argument(
+        '-n',
+        '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write to any repositories',
+    )
+    global_group.add_argument(
+        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
+    )
+    global_group.add_argument(
+        '-v',
+        '--verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to the console (from none to lots: 0, 1, or 2)',
+    )
+    global_group.add_argument(
+        '--syslog-verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive',
+    )
+    global_group.add_argument(
+        '--version',
+        dest='version',
+        default=False,
+        action='store_true',
+        help='Display installed version number of borgmatic and exit',
+    )
+
+    top_level_parser = ArgumentParser(
+        description='''
+            A simple wrapper script for the Borg backup software that creates and prunes backups.
+            If none of the action options are given, then borgmatic defaults to: prune, create, and
+            check archives.
+            ''',
+        parents=[global_parser],
+    )
+
+    subparsers = top_level_parser.add_subparsers(
+        title='actions',
+        metavar='',
+        help='Specify zero or more actions. Defaults to prune, create, and check. Use --help with action for details:',
+    )
+    init_parser = subparsers.add_parser(
+        'init',
+        aliases=SUBPARSER_ALIASES['init'],
+        help='Initialize an empty Borg repository',
+        description='Initialize an empty Borg repository',
+        add_help=False,
+    )
+    init_group = init_parser.add_argument_group('init arguments')
+    init_group.add_argument(
+        '-e',
+        '--encryption',
+        dest='encryption_mode',
+        help='Borg repository encryption mode',
+        required=True,
+    )
+    init_group.add_argument(
+        '--append-only',
+        dest='append_only',
+        action='store_true',
+        help='Create an append-only repository',
+    )
+    init_group.add_argument(
+        '--storage-quota',
+        dest='storage_quota',
+        help='Create a repository with a fixed storage quota',
+    )
+    init_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    prune_parser = subparsers.add_parser(
+        'prune',
+        aliases=SUBPARSER_ALIASES['prune'],
+        help='Prune archives according to the retention policy',
+        description='Prune archives according to the retention policy',
+        add_help=False,
+    )
+    prune_group = prune_parser.add_argument_group('prune arguments')
+    prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    create_parser = subparsers.add_parser(
+        'create',
+        aliases=SUBPARSER_ALIASES['create'],
+        help='Create archives (actually perform backups)',
+        description='Create archives (actually perform backups)',
+        add_help=False,
+    )
+    create_group = create_parser.add_argument_group('create arguments')
+    create_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    create_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    check_parser = subparsers.add_parser(
+        'check',
+        aliases=SUBPARSER_ALIASES['check'],
+        help='Check archives for consistency',
+        description='Check archives for consistency',
+        add_help=False,
+    )
+    check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    extract_parser = subparsers.add_parser(
+        'extract',
+        aliases=SUBPARSER_ALIASES['extract'],
+        help='Extract a named archive to the current directory',
+        description='Extract a named archive to the current directory',
+        add_help=False,
+    )
+    extract_group = extract_parser.add_argument_group('extract arguments')
+    extract_group.add_argument(
+        '--repository',
+        help='Path of repository to extract, defaults to the configured repository if there is only one',
+    )
+    extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    extract_group.add_argument(
+        '--restore-path',
+        nargs='+',
+        dest='restore_paths',
+        help='Paths to restore from archive, defaults to the entire archive',
+    )
+    extract_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    extract_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    list_parser = subparsers.add_parser(
+        'list',
+        aliases=SUBPARSER_ALIASES['list'],
+        help='List archives',
+        description='List archives or the contents of an archive',
+        add_help=False,
+    )
+    list_group = list_parser.add_argument_group('list arguments')
+    list_group.add_argument(
+        '--repository',
+        help='Path of repository to list, defaults to the configured repository if there is only one',
+    )
+    list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument(
+        '--short', default=False, action='store_true', help='Output only archive or path names'
+    )
+    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument(
+        '--json', default=False, action='store_true', help='Output results as JSON'
+    )
+    list_group.add_argument(
+        '-P', '--prefix', help='Only list archive names starting with this prefix'
+    )
+    list_group.add_argument(
+        '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
+    )
+    list_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    list_group.add_argument(
+        '--first', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '--last', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+    )
+    list_group.add_argument(
+        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+    )
+    list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
+    list_group.add_argument(
+        '--pattern-from',
+        metavar='FILENAME',
+        help='Include or exclude paths matching patterns from pattern file, one per line',
+    )
+    list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    info_parser = subparsers.add_parser(
+        'info',
+        aliases=SUBPARSER_ALIASES['info'],
+        help='Display summary information on archives',
+        description='Display summary information on archives',
+        add_help=False,
+    )
+    info_group = info_parser.add_argument_group('info arguments')
+    info_group.add_argument(
+        '--repository',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one',
+    )
+    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    info_group.add_argument(
+        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+    )
+    info_group.add_argument(
+        '-a',
+        '--glob-archives',
+        metavar='GLOB',
+        help='Only show info for archive names matching this glob',
+    )
+    info_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    info_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Show info for first N archives after other filters are applied',
+    )
+    info_group.add_argument(
+        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+    )
+    info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments = parse_subparser_arguments(unparsed_arguments, top_level_parser, subparsers)
+
+    if arguments['global'].excludes_filename:
+        raise ValueError(
+            'The --excludes option has been replaced with exclude_patterns in configuration'
+        )
+
+    if 'init' in arguments and arguments['global'].dry_run:
+        raise ValueError('The init action cannot be used with the --dry-run option')
+
+    if (
+        'list' in arguments
+        and 'info' in arguments
+        and arguments['list'].json
+        and arguments['info'].json
+    ):
+        raise ValueError('With the --json option, list and info actions cannot be used together')
+
+    return arguments

borgmatic/commands/borgmatic.py

@@ -1,262 +1,37 @@
-from argparse import ArgumentParser
 import collections
 import json
 import logging
 import os
-from subprocess import CalledProcessError
 import sys
+from subprocess import CalledProcessError
 
+import colorama
 import pkg_resources
 
-from borgmatic.borg import (
-    check as borg_check,
-    create as borg_create,
-    environment as borg_environment,
-    prune as borg_prune,
-    extract as borg_extract,
-    list as borg_list,
-    info as borg_info,
-    init as borg_init,
-)
-from borgmatic.commands import hook
+from borgmatic import hook
+from borgmatic.borg import check as borg_check
+from borgmatic.borg import create as borg_create
+from borgmatic.borg import environment as borg_environment
+from borgmatic.borg import extract as borg_extract
+from borgmatic.borg import info as borg_info
+from borgmatic.borg import init as borg_init
+from borgmatic.borg import list as borg_list
+from borgmatic.borg import prune as borg_prune
+from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
+from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
 
-
 logger = logging.getLogger(__name__)
 
-
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
 
 
-def parse_arguments(*arguments):
+def run_configuration(config_filename, config, arguments):  # pragma: no cover
     '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an argparse.ArgumentParser instance.
-    '''
-    config_paths = collect.get_default_config_paths()
-
-    parser = ArgumentParser(
-        description='''
-            A simple wrapper script for the Borg backup software that creates and prunes backups.
-            If none of the action options are given, then borgmatic defaults to: prune, create, and
-            check archives.
-            ''',
-        add_help=False,
-    )
-
-    actions_group = parser.add_argument_group('actions')
-    actions_group.add_argument(
-        '-I', '--init', dest='init', action='store_true', help='Initialize an empty Borg repository'
-    )
-    actions_group.add_argument(
-        '-p',
-        '--prune',
-        dest='prune',
-        action='store_true',
-        help='Prune archives according to the retention policy',
-    )
-    actions_group.add_argument(
-        '-C',
-        '--create',
-        dest='create',
-        action='store_true',
-        help='Create archives (actually perform backups)',
-    )
-    actions_group.add_argument(
-        '-k', '--check', dest='check', action='store_true', help='Check archives for consistency'
-    )
-
-    actions_group.add_argument(
-        '-x',
-        '--extract',
-        dest='extract',
-        action='store_true',
-        help='Extract a named archive to the current directory',
-    )
-    actions_group.add_argument(
-        '-l', '--list', dest='list', action='store_true', help='List archives'
-    )
-    actions_group.add_argument(
-        '-i',
-        '--info',
-        dest='info',
-        action='store_true',
-        help='Display summary information on archives',
-    )
-
-    init_group = parser.add_argument_group('options for --init')
-    init_group.add_argument(
-        '-e', '--encryption', dest='encryption_mode', help='Borg repository encryption mode'
-    )
-    init_group.add_argument(
-        '--append-only',
-        dest='append_only',
-        action='store_true',
-        help='Create an append-only repository',
-    )
-    init_group.add_argument(
-        '--storage-quota',
-        dest='storage_quota',
-        help='Create a repository with a fixed storage quota',
-    )
-
-    prune_group = parser.add_argument_group('options for --prune')
-    stats_argument = prune_group.add_argument(
-        '--stats',
-        dest='stats',
-        default=False,
-        action='store_true',
-        help='Display statistics of archive',
-    )
-
-    create_group = parser.add_argument_group('options for --create')
-    progress_argument = create_group.add_argument(
-        '--progress',
-        dest='progress',
-        default=False,
-        action='store_true',
-        help='Display progress for each file as it is processed',
-    )
-    create_group._group_actions.append(stats_argument)
-    json_argument = create_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
-    )
-
-    extract_group = parser.add_argument_group('options for --extract')
-    repository_argument = extract_group.add_argument(
-        '--repository',
-        help='Path of repository to use, defaults to the configured repository if there is only one',
-    )
-    archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
-    extract_group.add_argument(
-        '--restore-path',
-        nargs='+',
-        dest='restore_paths',
-        help='Paths to restore from archive, defaults to the entire archive',
-    )
-    extract_group._group_actions.append(progress_argument)
-
-    list_group = parser.add_argument_group('options for --list')
-    list_group._group_actions.append(repository_argument)
-    list_group._group_actions.append(archive_argument)
-    list_group._group_actions.append(json_argument)
-
-    info_group = parser.add_argument_group('options for --info')
-    info_group._group_actions.append(json_argument)
-
-    common_group = parser.add_argument_group('common options')
-    common_group.add_argument(
-        '-c',
-        '--config',
-        nargs='+',
-        dest='config_paths',
-        default=config_paths,
-        help='Configuration filenames or directories, defaults to: {}'.format(
-            ' '.join(config_paths)
-        ),
-    )
-    common_group.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        help='Deprecated in favor of exclude_patterns within configuration',
-    )
-    common_group.add_argument(
-        '-n',
-        '--dry-run',
-        dest='dry_run',
-        action='store_true',
-        help='Go through the motions, but do not actually write to any repositories',
-    )
-    common_group.add_argument(
-        '-v',
-        '--verbosity',
-        type=int,
-        choices=range(0, 3),
-        default=0,
-        help='Display verbose progress (1 for some, 2 for lots)',
-    )
-    common_group.add_argument(
-        '--version',
-        dest='version',
-        default=False,
-        action='store_true',
-        help='Display installed version number of borgmatic and exit',
-    )
-    common_group.add_argument('--help', action='help', help='Show this help information and exit')
-
-    args = parser.parse_args(arguments)
-
-    if args.excludes_filename:
-        raise ValueError(
-            'The --excludes option has been replaced with exclude_patterns in configuration'
-        )
-
-    if (args.encryption_mode or args.append_only or args.storage_quota) and not args.init:
-        raise ValueError(
-            'The --encryption, --append-only, and --storage-quota options can only be used with the --init option'
-        )
-
-    if args.init and args.dry_run:
-        raise ValueError('The --init option cannot be used with the --dry-run option')
-    if args.init and not args.encryption_mode:
-        raise ValueError('The --encryption option is required with the --init option')
-
-    if not args.extract:
-        if not args.list:
-            if args.repository:
-                raise ValueError(
-                    'The --repository option can only be used with the --extract and --list options'
-                )
-            if args.archive:
-                raise ValueError(
-                    'The --archive option can only be used with the --extract and --list options'
-                )
-        if args.restore_paths:
-            raise ValueError('The --restore-path option can only be used with the --extract option')
-    if args.extract and not args.archive:
-        raise ValueError('The --archive option is required with the --extract option')
-
-    if args.progress and not (args.create or args.extract):
-        raise ValueError(
-            'The --progress option can only be used with the --create and --extract options'
-        )
-
-    if args.json and not (args.create or args.list or args.info):
-        raise ValueError(
-            'The --json option can only be used with the --create, --list, or --info options'
-        )
-
-    if args.json and args.list and args.info:
-        raise ValueError(
-            'With the --json option, options --list and --info cannot be used together'
-        )
-
-    # If any of the action flags are explicitly requested, leave them as-is. Otherwise, assume
-    # defaults: Mutate the given arguments to enable the default actions.
-    if (
-        not args.init
-        and not args.prune
-        and not args.create
-        and not args.check
-        and not args.extract
-        and not args.list
-        and not args.info
-    ):
-        args.prune = True
-        args.create = True
-        args.check = True
-
-    if args.stats and not (args.create or args.prune):
-        raise ValueError('The --stats option can only be used when creating or pruning archives')
-
-    return args
-
-
-def run_configuration(config_filename, config, args):  # pragma: no cover
-    '''
-    Given a config filename and the corresponding parsed config dict, execute its defined pruning,
+    Given a config filename, the corresponding parsed config dict, and command-line arguments as a
+    dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
     backups, consistency checks, and/or other actions.
 
     Yield JSON output strings from executing any actions that produce JSON.
@@ -265,20 +40,25 @@ def run_configuration(config_filename, config, args):  # pragma: no cover
         config.get(section_name, {})
         for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
     )
+    global_arguments = arguments['global']
 
     try:
         local_path = location.get('local_path', 'borg')
         remote_path = location.get('remote_path')
         borg_environment.initialize(storage)
 
-        if args.create:
+        if 'create' in arguments:
             hook.execute_hook(
-                hooks.get('before_backup'), config_filename, 'pre-backup', args.dry_run
+                hooks.get('before_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'pre-backup',
+                global_arguments.dry_run,
             )
 
         for repository_path in location['repositories']:
             yield from run_actions(
-                args=args,
+                arguments=arguments,
                 location=location,
                 storage=storage,
                 retention=retention,
@@ -288,17 +68,35 @@ def run_configuration(config_filename, config, args):  # pragma: no cover
                 repository_path=repository_path,
             )
 
-        if args.create:
+        if 'create' in arguments:
             hook.execute_hook(
-                hooks.get('after_backup'), config_filename, 'post-backup', args.dry_run
+                hooks.get('after_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'post-backup',
+                global_arguments.dry_run,
             )
     except (OSError, CalledProcessError):
-        hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error', args.dry_run)
+        hook.execute_hook(
+            hooks.get('on_error'),
+            hooks.get('umask'),
+            config_filename,
+            'on-error',
+            global_arguments.dry_run,
+        )
         raise
 
 
 def run_actions(
-    *, args, location, storage, retention, consistency, local_path, remote_path, repository_path
+    *,
+    arguments,
+    location,
+    storage,
+    retention,
+    consistency,
+    local_path,
+    remote_path,
+    repository_path
 ):  # pragma: no cover
     '''
     Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
@@ -308,148 +106,202 @@ def run_actions(
     Yield JSON output strings from executing any actions that produce JSON.
     '''
     repository = os.path.expanduser(repository_path)
-    dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
-    if args.init:
+    global_arguments = arguments['global']
+    dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
+    if 'init' in arguments:
         logger.info('{}: Initializing repository'.format(repository))
         borg_init.initialize_repository(
             repository,
-            args.encryption_mode,
-            args.append_only,
-            args.storage_quota,
+            arguments['init'].encryption_mode,
+            arguments['init'].append_only,
+            arguments['init'].storage_quota,
             local_path=local_path,
             remote_path=remote_path,
         )
-    if args.prune:
+    if 'prune' in arguments:
         logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
         borg_prune.prune_archives(
-            args.dry_run,
+            global_arguments.dry_run,
             repository,
             storage,
             retention,
             local_path=local_path,
             remote_path=remote_path,
-            stats=args.stats,
+            stats=arguments['prune'].stats,
         )
-    if args.create:
+    if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
         json_output = borg_create.create_archive(
-            args.dry_run,
+            global_arguments.dry_run,
             repository,
             location,
             storage,
             local_path=local_path,
             remote_path=remote_path,
-            progress=args.progress,
-            stats=args.stats,
-            json=args.json,
+            progress=arguments['create'].progress,
+            stats=arguments['create'].stats,
+            json=arguments['create'].json,
         )
         if json_output:
             yield json.loads(json_output)
-    if args.check and checks.repository_enabled_for_checks(repository, consistency):
+    if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
             repository, storage, consistency, local_path=local_path, remote_path=remote_path
         )
-    if args.extract:
-        if args.repository is None or repository == args.repository:
-            logger.info('{}: Extracting archive {}'.format(repository, args.archive))
+    if 'extract' in arguments:
+        if arguments['extract'].repository is None or repository == arguments['extract'].repository:
+            logger.info(
+                '{}: Extracting archive {}'.format(repository, arguments['extract'].archive)
+            )
             borg_extract.extract_archive(
-                args.dry_run,
+                global_arguments.dry_run,
                 repository,
-                args.archive,
-                args.restore_paths,
+                arguments['extract'].archive,
+                arguments['extract'].restore_paths,
                 location,
                 storage,
                 local_path=local_path,
                 remote_path=remote_path,
-                progress=args.progress,
+                progress=arguments['extract'].progress,
             )
-    if args.list:
-        if args.repository is None or repository == args.repository:
+    if 'list' in arguments:
+        if arguments['list'].repository is None or repository == arguments['list'].repository:
             logger.info('{}: Listing archives'.format(repository))
             json_output = borg_list.list_archives(
                 repository,
                 storage,
-                args.archive,
+                list_arguments=arguments['list'],
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            if json_output:
+                yield json.loads(json_output)
+    if 'info' in arguments:
+        if arguments['info'].repository is None or repository == arguments['info'].repository:
+            logger.info('{}: Displaying summary info for archives'.format(repository))
+            json_output = borg_info.display_archives_info(
+                repository,
+                storage,
+                info_arguments=arguments['info'],
                 local_path=local_path,
                 remote_path=remote_path,
-                json=args.json,
             )
             if json_output:
                 yield json.loads(json_output)
-    if args.info:
-        logger.info('{}: Displaying summary info for archives'.format(repository))
-        json_output = borg_info.display_archives_info(
-            repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
-        )
-        if json_output:
-            yield json.loads(json_output)
 
 
-def collect_configuration_run_summary_logs(config_filenames, args):
+def load_configurations(config_filenames):
     '''
-    Given a sequence of configuration filenames and parsed command-line arguments as an
-    argparse.ArgumentParser instance, run each configuration file and yield a series of
-    logging.LogRecord instances containing summary information about each run.
-
-    As a side effect of running through these configuration files, output their JSON results, if
-    any, to stdout.
+    Given a sequence of configuration filenames, load and validate each configuration file. Return
+    the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
+    and sequence of logging.LogRecord instances containing any parse errors.
     '''
     # Dict mapping from config filename to corresponding parsed config dict.
     configs = collections.OrderedDict()
+    logs = []
 
     # Parse and load each configuration file.
     for config_filename in config_filenames:
         try:
-            logger.info('{}: Parsing configuration file'.format(config_filename))
             configs[config_filename] = validate.parse_configuration(
                 config_filename, validate.schema_filename()
             )
         except (ValueError, OSError, validate.Validation_error) as error:
-            yield logging.makeLogRecord(
-                dict(
-                    levelno=logging.CRITICAL,
-                    msg='{}: Error parsing configuration file'.format(config_filename),
-                )
+            logs.extend(
+                [
+                    logging.makeLogRecord(
+                        dict(
+                            levelno=logging.CRITICAL,
+                            levelname='CRITICAL',
+                            msg='{}: Error parsing configuration file'.format(config_filename),
+                        )
+                    ),
+                    logging.makeLogRecord(
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+                    ),
+                ]
             )
-            yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
 
+    return (configs, logs)
+
+
+def collect_configuration_run_summary_logs(configs, arguments):
+    '''
+    Given a dict of configuration filename to corresponding parsed configuration, and parsed
+    command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
+    each configuration file and yield a series of logging.LogRecord instances containing summary
+    information about each run.
+
+    As a side effect of running through these configuration files, output their JSON results, if
+    any, to stdout.
+    '''
     # Run cross-file validation checks.
-    if args.extract or (args.list and args.archive):
+    if 'extract' in arguments:
+        repository = arguments['extract'].repository
+    elif 'list' in arguments and arguments['list'].archive:
+        repository = arguments['list'].repository
+    else:
+        repository = None
+
+    if repository:
         try:
-            validate.guard_configuration_contains_repository(args.repository, configs)
+            validate.guard_configuration_contains_repository(repository, configs)
         except ValueError as error:
-            yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            )
             return
 
     # Execute the actions corresponding to each configuration file.
     json_results = []
     for config_filename, config in configs.items():
         try:
-            json_results.extend(list(run_configuration(config_filename, config, args)))
+            json_results.extend(list(run_configuration(config_filename, config, arguments)))
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.INFO,
+                    levelname='INFO',
                     msg='{}: Successfully ran configuration file'.format(config_filename),
                 )
             )
-        except (ValueError, OSError, CalledProcessError) as error:
+        except CalledProcessError as error:
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.CRITICAL,
+                    levelname='CRITICAL',
                     msg='{}: Error running configuration file'.format(config_filename),
                 )
             )
-            yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
+            )
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            )
+        except (ValueError, OSError) as error:
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.CRITICAL,
+                    levelname='CRITICAL',
+                    msg='{}: Error running configuration file'.format(config_filename),
+                )
+            )
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            )
 
     if json_results:
         sys.stdout.write(json.dumps(json_results))
 
-    if not config_filenames:
+    if not configs:
         yield logging.makeLogRecord(
             dict(
                 levelno=logging.CRITICAL,
-                msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
+                levelname='CRITICAL',
+                msg='{}: No configuration files found'.format(
+                    ' '.join(arguments['global'].config_paths)
+                ),
             )
         )
 
@@ -458,7 +310,8 @@ def exit_with_help_link():  # pragma: no cover
     '''
     Display a link to get help and exit with an error code.
     '''
-    logger.critical('\nNeed some help? https://torsion.org/borgmatic/#issues')
+    logger.critical('')
+    logger.critical('Need some help? https://torsion.org/borgmatic/#issues')
     sys.exit(1)
 
 
@@ -466,26 +319,44 @@ def main():  # pragma: no cover
     configure_signals()
 
     try:
-        args = parse_arguments(*sys.argv[1:])
+        arguments = parse_arguments(*sys.argv[1:])
     except ValueError as error:
-        logging.basicConfig(level=logging.CRITICAL, format='%(message)s')
+        configure_logging(logging.CRITICAL)
         logger.critical(error)
         exit_with_help_link()
+    except SystemExit as error:
+        if error.code == 0:
+            raise error
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv)))
+        exit_with_help_link()
 
-    logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
-
-    if args.version:
+    global_arguments = arguments['global']
+    if global_arguments.version:
         print(pkg_resources.require('borgmatic')[0].version)
         sys.exit(0)
 
-    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
+    configs, parse_logs = load_configurations(config_filenames)
+
+    colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
+    configure_logging(
+        verbosity_to_log_level(global_arguments.verbosity),
+        verbosity_to_log_level(global_arguments.syslog_verbosity),
+    )
+
     logger.debug('Ensuring legacy configuration is upgraded')
     convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
 
-    summary_logs = tuple(collect_configuration_run_summary_logs(config_filenames, args))
+    summary_logs = list(collect_configuration_run_summary_logs(configs, arguments))
 
-    logger.info('\nsummary:')
-    [logger.handle(log) for log in summary_logs if log.levelno >= logger.getEffectiveLevel()]
+    logger.info('')
+    logger.info('summary:')
+    [
+        logger.handle(log)
+        for log in parse_logs + summary_logs
+        if log.levelno >= logger.getEffectiveLevel()
+    ]
 
     if any(log.levelno == logging.CRITICAL for log in summary_logs):
         exit_with_help_link()

borgmatic/commands/convert_config.py

@@ -1,13 +1,12 @@
-from argparse import ArgumentParser
 import os
 import sys
 import textwrap
+from argparse import ArgumentParser
 
 from ruamel import yaml
 
 from borgmatic.config import convert, generate, legacy, validate
 
-
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'

borgmatic/commands/generate_config.py

@@ -1,9 +1,8 @@
-from argparse import ArgumentParser
 import sys
+from argparse import ArgumentParser
 
 from borgmatic.config import generate, validate
 
-
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
 

borgmatic/commands/hook.py

@@ -1,34 +0,0 @@
-import logging
-import subprocess
-
-
-logger = logging.getLogger(__name__)
-
-
-def execute_hook(commands, config_filename, description, dry_run):
-    '''
-
-    Given a list of hook commands to execute, a config filename, a hook description, and whether
-    this is a dry run, run the given commands. Or, don't run them if this is a dry run.
-    '''
-    if not commands:
-        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
-        return
-
-    dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
-
-    if len(commands) == 1:
-        logger.info(
-            '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
-        )
-    else:
-        logger.info(
-            '{}: Running {} commands for {} hook{}'.format(
-                config_filename, len(commands), description, dry_run_label
-            )
-        )
-
-    for command in commands:
-        logger.debug('{}: Hook command: {}'.format(config_filename, command))
-        if not dry_run:
-            subprocess.check_call(command, shell=True)

borgmatic/commands/validate_config.py

@@ -0,0 +1,56 @@
+import logging
+import sys
+from argparse import ArgumentParser
+
+from borgmatic.config import collect, validate
+
+logger = logging.getLogger(__name__)
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
+    parser.add_argument(
+        '-c',
+        '--config',
+        nargs='+',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():  # pragma: no cover
+    args = parse_arguments(*sys.argv[1:])
+
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    if len(config_filenames) == 0:
+        logger.critical('No files to validate found')
+        sys.exit(1)
+
+    found_issues = False
+    for config_filename in config_filenames:
+        try:
+            validate.parse_configuration(config_filename, validate.schema_filename())
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logging.critical('{}: Error parsing configuration file'.format(config_filename))
+            logging.critical(error)
+            found_issues = True
+
+    if found_issues:
+        sys.exit(1)
+    else:
+        logger.info(
+            'All given configuration files are valid: {}'.format(', '.join(config_filenames))
+        )

borgmatic/config/collect.py

@@ -22,7 +22,7 @@ def collect_config_filenames(config_paths):
     '''
     Given a sequence of config paths, both filenames and directories, resolve that to an iterable
     of files. Accomplish this by listing any given directories looking for contained config files
-    (ending with the ".yaml" extension). This is non-recursive, so any directories within the given
+    (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given
     directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
@@ -43,5 +43,6 @@ def collect_config_filenames(config_paths):
 
         for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
-            if full_filename.endswith('.yaml') and not os.path.isdir(full_filename):
+            matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
+            if matching_filetype and not os.path.isdir(full_filename):
                 yield full_filename

borgmatic/config/convert.py

@@ -64,9 +64,9 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
     return destination_config
 
 
-class LegacyConfigurationNotUpgraded(FileNotFoundError):
+class Legacy_configuration_not_upgraded(FileNotFoundError):
     def __init__(self):
-        super(LegacyConfigurationNotUpgraded, self).__init__(
+        super(Legacy_configuration_not_upgraded, self).__init__(
             '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
 to YAML. This better supports validation, and has a more natural way to express
 lists of values. To upgrade your existing configuration, run:
@@ -83,7 +83,7 @@ instead of the old one.'''
 def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
     '''
     If legacy source configuration exists but no destination upgraded configs do, raise
-    LegacyConfigurationNotUpgraded.
+    Legacy_configuration_not_upgraded.
 
     The idea is that we want to alert the user about upgrading their config if they haven't already.
     '''
@@ -92,4 +92,4 @@ def guard_configuration_upgraded(source_config_filename, destination_config_file
     )
 
     if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise LegacyConfigurationNotUpgraded()
+        raise Legacy_configuration_not_upgraded()

borgmatic/config/generate.py

@@ -2,7 +2,6 @@ import os
 
 from ruamel import yaml
 
-
 INDENT = 4
 
 

borgmatic/config/legacy.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict, namedtuple
 from configparser import RawConfigParser
 
-
 Section_format = namedtuple('Section_format', ('name', 'options'))
 Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
 

borgmatic/config/load.py

@@ -1,9 +1,8 @@
-import os
 import logging
+import os
 
 import ruamel.yaml
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -54,9 +53,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor):
 
         for index, (key_node, value_node) in enumerate(node.value):
             if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
-                included_value = representer.represent_mapping(
-                    tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
-                )
+                included_value = representer.represent_data(self.construct_object(value_node))
                 node.value[index] = (key_node, included_value)
 
         super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -36,6 +36,18 @@ map:
                 type: bool
                 desc: Only store/extract numeric user and group identifiers. Defaults to false.
                 example: true
+            atime:
+                type: bool
+                desc: Store atime into archive. Defaults to true.
+                example: false
+            ctime:
+                type: bool
+                desc: Store ctime into archive. Defaults to true.
+                example: false
+            birthtime:
+                type: bool
+                desc: Store birthtime (creation date) into archive. Defaults to true.
+                example: false
             read_special:
                 type: bool
                 desc: |
@@ -156,7 +168,7 @@ map:
                 desc: |
                     Type of compression to use when creating archives. See
                     https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
-                    Defaults to no compression.
+                    Defaults to "lz4".
                 example: lz4
             remote_rate_limit:
                 type: int
@@ -168,6 +180,32 @@ map:
                     Command to use instead of "ssh". This can be used to specify ssh options.
                     Defaults to not set.
                 example: ssh -i /path/to/private/key
+            borg_base_directory:
+                type: scalar
+                desc: |
+                    Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
+                    See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
+                example: /path/to/base
+            borg_config_directory:
+                type: scalar
+                desc: |
+                    Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
+                example: /path/to/base/config
+            borg_cache_directory:
+                type: scalar
+                desc: |
+                    Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
+                example: /path/to/base/cache
+            borg_security_directory:
+                type: scalar
+                desc: |
+                    Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
+                example: /path/to/base/config/security
+            borg_keys_directory:
+                type: scalar
+                desc: |
+                    Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
+                example: /path/to/base/config/keys
             umask:
                 type: scalar
                 desc: Umask to be used for borg create. Defaults to 0077.
@@ -190,7 +228,9 @@ map:
         desc: |
             Retention policy for how many backups to keep in each category. See
             https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
-            At least one of the "keep" options is required for pruning to work.
+            At least one of the "keep" options is required for pruning to work. See
+            https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/
+            if you'd like to skip pruning entirely.
         map:
             keep_within:
                 type: scalar
@@ -273,6 +313,16 @@ map:
                     this prefix. Borg placeholders can be used. See the output of
                     "borg help placeholders" for details. Defaults to "{hostname}-".
                 example: sourcehostname
+    output:
+        desc: |
+            Options for customizing borgmatic's own output and logging.
+        map:
+            color:
+                type: bool
+                desc: |
+                    Apply color to console output. Can be overridden with --no-color command-line
+                    flag. Defaults to true.
+                example: false
     hooks:
         desc: |
             Shell commands or scripts to execute before and after a backup or if an error has occurred.
@@ -298,3 +348,7 @@ map:
                 desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
                 example:
                     - echo "Error while creating a backup."
+            umask:
+                type: scalar
+                desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.
+                example: 0077

borgmatic/config/validate.py

@@ -8,9 +8,6 @@ import ruamel.yaml
 from borgmatic.config import load
 
 
-logger = logging.getLogger(__name__)
-
-
 def schema_filename():
     '''
     Path to the installed YAML configuration schema file, used to validate and parse the
@@ -66,13 +63,6 @@ def apply_logical_validation(config_filename, parsed_configuration):
                 ),
             )
 
-    consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
-    if archive_name_format and not consistency_prefix:
-        logger.warning(
-            'Since version 1.1.16, if you provide `archive_name_format`, you should also'
-            ' specify `consistency.prefix`.'
-        )
-
 
 def parse_configuration(config_filename, schema_filename):
     '''

borgmatic/execute.py

@@ -0,0 +1,57 @@
+import logging
+import subprocess
+
+logger = logging.getLogger(__name__)
+
+
+ERROR_OUTPUT_MAX_LINE_COUNT = 25
+
+
+def execute_and_log_output(full_command, output_log_level, shell):
+    last_lines = []
+    process = subprocess.Popen(
+        full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
+    )
+
+    while process.poll() is None:
+        line = process.stdout.readline().rstrip().decode()
+        if not line:
+            continue
+
+        # Keep the last few lines of output in case the command errors, and we need the output for
+        # the exception below.
+        last_lines.append(line)
+        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.pop(0)
+
+        logger.log(output_log_level, line)
+
+    remaining_output = process.stdout.read().rstrip().decode()
+    if remaining_output:  # pragma: no cover
+        logger.log(output_log_level, remaining_output)
+
+    exit_code = process.poll()
+    if exit_code != 0:
+        # If an error occurs, include its output in the raised exception so that we don't
+        # inadvertently hide error output.
+        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.insert(0, '...')
+
+        raise subprocess.CalledProcessError(
+            exit_code, ' '.join(full_command), '\n'.join(last_lines)
+        )
+
+
+def execute_command(full_command, output_log_level=logging.INFO, shell=False):
+    '''
+    Execute the given command (a sequence of command/argument strings) and log its output at the
+    given log level. If output log level is None, instead capture and return the output. If
+    shell is True, execute the command within a shell.
+    '''
+    logger.debug(' '.join(full_command))
+
+    if output_log_level is None:
+        output = subprocess.check_output(full_command, shell=shell)
+        return output.decode() if output is not None else None
+    else:
+        execute_and_log_output(full_command, output_log_level, shell=shell)

borgmatic/hook.py

@@ -0,0 +1,53 @@
+import logging
+import os
+
+from borgmatic import execute
+
+logger = logging.getLogger(__name__)
+
+
+def execute_hook(commands, umask, config_filename, description, dry_run):
+    '''
+    Given a list of hook commands to execute, a umask to execute with (or None), a config filename,
+    a hook description, and whether this is a dry run, run the given commands. Or, don't run them
+    if this is a dry run.
+
+    Raise ValueError if the umask cannot be parsed.
+    '''
+    if not commands:
+        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
+        return
+
+    dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
+
+    if len(commands) == 1:
+        logger.info(
+            '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
+        )
+    else:
+        logger.info(
+            '{}: Running {} commands for {} hook{}'.format(
+                config_filename, len(commands), description, dry_run_label
+            )
+        )
+
+    if umask:
+        parsed_umask = int(str(umask), 8)
+        logger.debug('{}: Set hook umask to {}'.format(config_filename, oct(parsed_umask)))
+        original_umask = os.umask(parsed_umask)
+    else:
+        original_umask = None
+
+    try:
+        for command in commands:
+            if not dry_run:
+                execute.execute_command(
+                    [command],
+                    output_log_level=logging.ERROR
+                    if description == 'on-error'
+                    else logging.WARNING,
+                    shell=True,
+                )
+    finally:
+        if original_umask:
+            os.umask(original_umask)

borgmatic/logger.py

@@ -0,0 +1,101 @@
+import logging
+import os
+import sys
+
+import colorama
+
+
+def to_bool(arg):
+    '''
+    Return a boolean value based on `arg`.
+    '''
+    if arg is None or isinstance(arg, bool):
+        return arg
+
+    if isinstance(arg, str):
+        arg = arg.lower()
+
+    if arg in ('yes', 'on', '1', 'true', 1):
+        return True
+
+    return False
+
+
+def interactive_console():
+    '''
+    Return whether the current console is "interactive". Meaning: Capable of
+    user input and not just something like a cron job.
+    '''
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+
+
+def should_do_markup(no_color, configs):
+    '''
+    Given the value of the command-line no-color argument, and a dict of configuration filename to
+    corresponding parsed configuration, determine if we should enable colorama marking up.
+    '''
+    if no_color:
+        return False
+
+    if any(config.get('output', {}).get('color') is False for config in configs.values()):
+        return False
+
+    py_colors = os.environ.get('PY_COLORS', None)
+
+    if py_colors is not None:
+        return to_bool(py_colors)
+
+    return interactive_console()
+
+
+LOG_LEVEL_TO_COLOR = {
+    logging.CRITICAL: colorama.Fore.RED,
+    logging.ERROR: colorama.Fore.RED,
+    logging.WARN: colorama.Fore.YELLOW,
+    logging.INFO: colorama.Fore.GREEN,
+    logging.DEBUG: colorama.Fore.CYAN,
+}
+
+
+class Console_color_formatter(logging.Formatter):
+    def format(self, record):
+        color = LOG_LEVEL_TO_COLOR.get(record.levelno)
+        return color_text(color, record.msg)
+
+
+def color_text(color, message):
+    '''
+    Give colored text.
+    '''
+    if not color:
+        return message
+
+    return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
+
+
+def configure_logging(console_log_level, syslog_log_level=None):
+    '''
+    Configure logging to go to both the console and syslog. Use the given log levels, respectively.
+    '''
+    if syslog_log_level is None:
+        syslog_log_level = console_log_level
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(Console_color_formatter())
+    console_handler.setLevel(console_log_level)
+
+    syslog_path = None
+    if os.path.exists('/dev/log'):
+        syslog_path = '/dev/log'
+    elif os.path.exists('/var/run/syslog'):
+        syslog_path = '/var/run/syslog'
+
+    if syslog_path and not interactive_console():
+        syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
+        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
+        syslog_handler.setLevel(syslog_log_level)
+        handlers = (console_handler, syslog_handler)
+    else:
+        handlers = (console_handler,)
+
+    logging.basicConfig(level=min(console_log_level, syslog_log_level), handlers=handlers)

borgmatic/verbosity.py

@@ -1,6 +1,5 @@
 import logging
 
-
 VERBOSITY_WARNING = 0
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2

docs/Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.7.3-alpine3.9 as borgmatic
+
+COPY . /app
+RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
+RUN borgmatic --help > /command-line.txt \
+    && for action in init prune create check extract list info; do \
+           echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
+           && borgmatic "$action" --help >> /command-line.txt; done
+
+FROM node:11.15.0-alpine as html
+
+WORKDIR /source
+
+RUN npm install @11ty/eleventy \
+    @11ty/eleventy-plugin-syntaxhighlight \
+    @11ty/eleventy-plugin-inclusive-language \
+    markdown-it \
+    markdown-it-anchor \
+    markdown-it-replace-link
+COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml
+COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt
+COPY . /source
+RUN npx eleventy --input=/source/docs --output=/output/docs \
+  && mv /output/docs/index.html /output/index.html
+
+FROM nginx:1.16.0-alpine
+
+COPY --from=html /output /usr/share/nginx/html
+COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/README.md

@@ -0,0 +1 @@
+../README.md

docs/_data/layout.json

@@ -0,0 +1 @@
+"layouts/main.njk"

docs/_includes/asciinema.css

@@ -0,0 +1,3 @@
+.asciicast > iframe {
+    width: 100% !important;
+}

docs/_includes/components/external-links.css

@@ -0,0 +1,12 @@
+/* External links */
+a[href^="http://"]:not(.minilink):not(.elv-externalexempt),
+a[href^="https://"]:not(.minilink):not(.elv-externalexempt), 
+a[href^="//"]:not(.minilink):not(.elv-externalexempt) {
+	text-decoration-color: inherit;
+}
+/* External link hovers */
+a[href^="http://"]:not(.minilink):not(.elv-externalexempt):hover,
+a[href^="https://"]:not(.minilink):not(.elv-externalexempt):hover, 
+a[href^="//"]:not(.minilink):not(.elv-externalexempt):hover {
+	text-decoration-color: #00bcd4;
+}

docs/_includes/components/info-blocks.css

@@ -0,0 +1,34 @@
+/* Warning */
+.elv-info {
+	line-height: 1.5;
+	padding: 0.8125em 1em 0.75em; /* 13px 16px 12px /16 */
+	margin-left: -1rem;
+	margin-right: -1rem;
+	margin-bottom: 2em;
+	background-color: #dff7ff;
+}
+.elv-info:before {
+	content: "ℹ️ ";
+}
+.elv-info-warn {
+	background-color: #ffa;
+}
+.elv-info-warn:before {
+	content: "⚠️ ";
+}
+.elv-info:first-child {
+	margin-top: 0;
+}
+body > .elv-info {
+	margin-left: 0;
+	margin-right: 0;
+	padding: .5rem 1rem;
+}
+@media (min-width: 37.5em) and (min-height: 25em) { /* 600px / 400px */
+	body > .elv-info-sticky {
+		position: sticky;
+		top: 0;
+		z-index: 2;
+		box-shadow: 0 3px 0 0 rgba(0,0,0,.08);
+	}
+}

docs/_includes/components/lists.css

@@ -0,0 +1,126 @@
+/* Buzzwords */
+@keyframes rainbow {
+	0% { background-position: 0% 50%; }
+	50% { background-position: 100% 50%; }
+	100% { background-position: 0% 50%; }
+}
+.buzzword-list,
+.inlinelist {
+	padding: 0;
+}
+.inlinelist:first-child:last-child {
+	margin: 0;
+}
+.buzzword,
+.buzzword-list li,
+.inlinelist .inlinelist-item {
+	display: inline;
+	-webkit-box-decoration-break: clone;
+	box-decoration-break: clone;
+	font-family: Georgia, serif;
+	font-size: 116%;
+	white-space: normal;
+	line-height: 1.85;
+	padding: .2em .5em;
+	margin: 4px 4px 4px 0;
+	transition: .15s linear outline;
+}
+.inlinelist .inlinelist-item.active {
+	background-color: #222;
+	color: #fff;
+	font-weight: inherit;
+}
+.inlinelist .inlinelist-item.active :link,
+.inlinelist .inlinelist-item.active :visited {
+	color: #fff;
+}
+.inlinelist .inlinelist-item code {
+	background-color: transparent;
+}
+a.buzzword {
+	text-decoration: underline;
+}
+.buzzword-list a,
+.inlinelist a {
+	text-decoration: none;
+}
+.inlinelist .inlinelist-item {
+	font-size: 100%;
+	line-height: 2;
+}
+@supports not(-webkit-box-decoration-break: clone) {
+	.buzzword,
+	.buzzword-list li,
+	.inlinelist .inlinelist-item {
+		display: inline-block;
+	}
+}
+.buzzword-list li,
+.buzzword {
+	background-color: #f7f7f7;
+}
+.inlinelist .inlinelist-item {
+	background-color: #e9e9e9;
+}
+.inlinelist .inlinelist-item:hover,
+.inlinelist .inlinelist-item:focus,
+.buzzword-list li:hover,
+.buzzword-list li:focus,
+.buzzword:hover,
+.buzzword:focus {
+	position: relative;
+	background-image: linear-gradient(238deg, #ff0000, #ff8000, #ffff00, #80ff00, #00ff00, #00ff80, #00ffff, #0080ff, #0000ff, #8000ff, #ff0080);
+	background-size: 1200% 1200%;
+	color: #fff;
+	text-shadow: 0 0 2px rgba(0,0,0,.9);
+	animation: rainbow 1.6s infinite;
+}
+.inlinelist .inlinelist-item:hover a,
+.inlinelist .inlinelist-item:focus a,
+.buzzword-list li:hover a,
+.buzzword-list li:focus a,
+a.buzzword:hover,
+a.buzzword:focus {
+	color: #fff;
+	text-decoration: none;
+}
+/*
+I wish there were a PE friendly way to do this but media queries don’t work work with @supports
+@media (prefers-reduced-motion: no-preference) {
+	.buzzword:hover,
+	.buzzword:focus {
+		animation: rainbow 1s infinite;
+	}
+}*/
+.buzzword-list li:hover:after,
+.buzzword-list li:focus:after,
+.buzzword:hover:after,
+.buzzword:focus:after {
+	font-family: system-ui, sans-serif;
+	content: "Buzzword alert!!!";
+	position: absolute;
+	left: 0;
+	top: 0;
+	max-width: 8em;
+	color: #f00;
+	font-weight: 700;
+	text-transform: uppercase;
+	transform: rotate(-10deg) translate(-25%, -125%);
+	text-shadow: 1px 1px 5px rgba(0,0,0,.6);
+	line-height: 1.2;
+	pointer-events: none;
+}
+main h2 .buzzword,
+main h3 .buzzword,
+main p .buzzword {
+	padding: 0px 7px;
+	font-size: 1em; /* 18px /18 */
+	margin: 0;
+	line-height: 1.444444444444; /* 26px /18 */
+	font-family: inherit;
+}
+main h2 a.buzzword,
+main h3 a.buzzword,
+main p a.buzzword {
+	text-decoration: underline;
+}

docs/_includes/components/minilink.css

@@ -0,0 +1,40 @@
+/* Mini link */
+.minilink {
+	display: inline-block;
+	padding: .125em .375em;
+	text-transform: uppercase;
+	font-size: 0.875rem; /* 14px /16 */
+	text-decoration: none;
+	background-color: #ddd;
+	border-radius: 0.1875em; /* 3px /16 */
+	font-weight: 500;
+	margin: 0 0.4285714285714em 0.07142857142857em 0; /* 0 6px 1px 0 /14 */
+	line-height: 1.285714285714; /* 18px /14 */
+	font-family: system-ui, sans-serif;
+}
+.minilink[href] {
+	box-shadow: 0 1px 1px 0 rgba(0,0,0,.5);
+}
+.minilink[href]:hover,
+.minilink[href]:focus {
+	background-color: #bbb;
+}
+pre + .minilink {
+	color: #fff;
+	border-radius: 0 0 0.2857142857143em 0.2857142857143em; /* 4px /14 */
+	float: right;
+	background-color: #444;
+	color: #fff;
+}
+pre[class*=language-] + .minilink {
+	position: relative;
+	top: -0.7142857142857em; /* -10px /14 */
+}
+p.minilink {
+	float: right;
+	margin-left: 2em;
+	margin-bottom: 2em;
+}
+.minilink + pre[class*=language-] {
+	clear: both;
+}

docs/_includes/components/toc.css

@@ -0,0 +1,63 @@
+.elv-toc {
+	font-size: 1rem; /* Reset */
+}
+@media (min-width: 64em) { /* 1024px */
+	.elv-toc {
+		position: absolute;
+		left: -17rem;
+		width: 16rem;
+	}
+}
+
+
+.elv-toc-list {
+	padding-left: 0;
+	padding-right: 0;
+	list-style: none;
+}
+/* Nested lists */
+.elv-toc-list ul {
+	padding: 0;
+	display: none;
+	margin-bottom: 1.5em;
+	list-style: none;
+}
+.elv-toc-list ul li {
+	padding-left: 0.875em; /* 14px /16 */
+}
+@media (min-width: 64em) and (min-height: 48em) { /* 1024 x 768px */
+	.elv-toc-list ul {
+		display: block;
+	}
+}
+
+/* List items */
+.elv-toc-list a:not(:hover) {
+	text-decoration: none;
+}
+.elv-toc-list li {
+	padding-top: 0;
+	padding-bottom: 0;
+	margin: .1em 0 .5em;
+}
+/* Top level links */
+.elv-toc-list > li > a {
+	font-weight: 400;
+	font-size: 1.0625em; /* 17px /16 */
+	color: #222;
+}
+
+/* Active links */
+.elv-toc-list li.elv-toc-active > a {
+	font-weight: 700;
+	text-decoration: underline;
+}
+.elv-toc-active > a:after {
+	content: " ⬅";
+	line-height: .5;
+}
+/* Show only active nested lists */
+.elv-toc-list ul.elv-toc-active,
+.elv-toc-list li.elv-toc-active > ul {
+	display: block;
+}

docs/_includes/header.njk

@@ -0,0 +1,3 @@
+<header class="elv-layout elv-layout-full elv-header{% if headerClass %} {{ headerClass }}{% endif %}">
+	<h1 class="elv-hed">{{ title | safe }}</h1>
+</header>

docs/_includes/layouts/base.njk

@@ -0,0 +1,27 @@
+<!doctype html>
+<html lang="en"{% if templateClass %} class="{{ templateClass }}"{% endif %}>
+	<head>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<title>{{ subtitle + ' - ' if subtitle}}{{ title }}</title>
+{%- set css %}
+{% include 'index.css' %}
+{% include 'components/lists.css' %}
+{% include 'components/external-links.css' %}
+{% include 'components/minilink.css' %}
+{% include 'components/toc.css' %}
+{% include 'components/info-blocks.css' %}
+{% include 'prism-theme.css' %}
+{% include 'asciinema.css' %}
+{% endset %}
+		<style>{{ css | safe }}</style>
+{% if feedTitle and feedUrl %}
+		<link rel="alternate" href="{{ feedUrl }}" title="{{ feedTitle }}" type="application/atom+xml">
+{% endif %}
+	</head>
+	<body>
+
+		{{ content | safe }}
+
+	</body>
+</html>

docs/_includes/layouts/main.njk

@@ -0,0 +1,12 @@
+---
+layout: layouts/base.njk
+templateClass: elv-default
+headerClass: elv-header-default
+---
+{% include "header.njk" %}
+
+<main class="elv-layout{% if layoutClass %} {{ layoutClass }}{% endif %}">
+    <article>
+        {{ content | safe }}
+    </article>
+</main>

docs/_includes/prism-theme.css

@@ -0,0 +1,172 @@
+/**
+ * prism.js default theme for JavaScript, CSS and HTML
+ * Based on dabblet (http://dabblet.com)
+ * @author Lea Verou
+ */
+code[class*="language-"],
+pre[class*="language-"] {
+  color: #ABB2BF;
+  background: none;
+  font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
+  text-align: left;
+  white-space: pre;
+  word-spacing: normal;
+  word-break: normal;
+  word-wrap: normal;
+  line-height: 1.5;
+  -moz-tab-size: 4;
+  -o-tab-size: 4;
+  tab-size: 4;
+  -webkit-hyphens: none;
+  -moz-hyphens: none;
+  -ms-hyphens: none;
+  hyphens: none;
+}
+
+pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection,
+code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection {
+  text-shadow: none;
+  background: #383e49;
+}
+
+pre[class*="language-"]::selection, pre[class*="language-"] ::selection,
+code[class*="language-"]::selection, code[class*="language-"] ::selection {
+  text-shadow: none;
+  background: #9aa2b1;
+}
+
+@media print {
+  code[class*="language-"],
+  pre[class*="language-"] {
+    text-shadow: none;
+  }
+}
+/* Code blocks */
+pre[class*="language-"] {
+  padding: 1em;
+  margin: .5em 0;
+  overflow: auto;
+}
+
+:not(pre) > code[class*="language-"],
+pre[class*="language-"] {
+  background: #282c34;
+}
+
+/* Inline code */
+:not(pre) > code[class*="language-"] {
+  padding: .1em;
+  border-radius: .3em;
+  white-space: normal;
+}
+
+.token.comment,
+.token.prolog,
+.token.doctype,
+.token.cdata {
+  color: #5C6370;
+}
+
+.token.punctuation {
+  color: #abb2bf;
+}
+
+.token.selector,
+.token.tag {
+  color: #e06c75;
+}
+
+.token.property,
+.token.boolean,
+.token.number,
+.token.constant,
+.token.symbol,
+.token.attr-name,
+.token.deleted {
+  color: #d19a66;
+}
+
+.token.string,
+.token.char,
+.token.attr-value,
+.token.builtin,
+.token.inserted {
+  color: #98c379;
+}
+
+.token.operator,
+.token.entity,
+.token.url,
+.language-css .token.string,
+.style .token.string {
+  color: #56b6c2;
+}
+
+.token.atrule,
+.token.keyword {
+  color: #e06c75;
+}
+
+.token.function {
+  color: #61afef;
+}
+
+.token.regex,
+.token.important,
+.token.variable {
+  color: #c678dd;
+}
+
+.token.important,
+.token.bold {
+  font-weight: bold;
+}
+
+.token.italic {
+  font-style: italic;
+}
+
+.token.entity {
+  cursor: help;
+}
+
+pre.line-numbers {
+	position: relative;
+	padding-left: 3.8em;
+	counter-reset: linenumber;
+}
+
+pre.line-numbers > code {
+	position: relative;
+}
+
+.line-numbers .line-numbers-rows {
+	position: absolute;
+	pointer-events: none;
+	top: 0;
+	font-size: 100%;
+	left: -3.8em;
+	width: 3em; /* works for line-numbers below 1000 lines */
+	letter-spacing: -1px;
+	border-right: 0;
+
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+
+}
+
+.line-numbers-rows > span {
+	pointer-events: none;
+	display: block;
+	counter-increment: linenumber;
+}
+
+.line-numbers-rows > span:before {
+	content: counter(linenumber);
+	color: #5C6370;
+	display: block;
+	padding-right: 0.8em;
+	text-align: right;
+}

docs/how-to/deal-with-very-large-backups.md

@@ -17,15 +17,18 @@ run borgmatic's pruning, creating, or checking actions separately. For
 instance, the the following optional flags are available:
 
 ```bash
-borgmatic --prune
-borgmatic --create
-borgmatic --check
+borgmatic prune
+borgmatic create
+borgmatic check
 ```
 
+(No borgmatic `prune`, `create`, or `check` actions? Try the old-style
+`--prune`, `--create`, or `--check`. Or upgrade borgmatic!)
+
 You can run with only one of these flags provided, or you can mix and match
 any number of them in a single borgmatic run. This supports approaches like
-making backups with `--create` on a frequent schedule, while only running
-expensive consistency checks with `--check` on a much less frequent basis from
+making backups with `create` on a frequent schedule, while only running
+expensive consistency checks with `check` on a much less frequent basis from
 a separate cron job.
 
 ### Consistency check configuration
@@ -93,4 +96,4 @@ backups.
 
 ## Related documentation
 
- * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)

docs/how-to/develop-on-borgmatic.md

@@ -41,7 +41,7 @@ you're in the `borgmatic/` working copy, install tox, which is used for
 setting up testing environments:
 
 ```bash
-sudo pip3 install tox
+pip3 install --user tox
 ```
 
 Finally, to actually run tests, run:
@@ -62,6 +62,14 @@ following:
 tox -e black
 ```
 
+And if you get a complaint from the
+[isort](https://github.com/timothycrosley/isort) Python import orderer, you
+can ask isort to order your imports for you:
+
+```bash
+tox -e isort
+```
+
 ### End-to-end tests
 
 borgmatic additionally includes some end-to-end tests that integration test
@@ -87,11 +95,18 @@ the following deviations from it:
    indentation with an opening delimeter.
 
 borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
-formatter and [Flake8](http://flake8.pycqa.org/en/latest/) code checker, so
+formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
+the [isort](https://github.com/timothycrosley/isort) import orderer, so
 certain code style requirements will be enforced when running automated tests.
-See the Black and Flake8 documentation for more information.
+See the Black, Flake8, and isort documentation for more information.
 
+## Continuous integration
+
+Each pull request triggers a continuous integration build which runs the test
+suite. You can view these builds on
+[build.torsion.org](https://build.torsion.org/witten/borgmatic), and they're
+also linked from the commits list on each pull request.
 
 ## Related documentation
 
- * [Inspect your backups](../../docs/how-to/inspect-your-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)

docs/how-to/inspect-your-backups.md

@@ -29,18 +29,61 @@ functionality:
 
 
 ```bash
-borgmatic --list
-borgmatic --info
+borgmatic list
+borgmatic info
 ```
 
+(No borgmatic `list` or `info` actions? Try the old-style `--list` or
+`--info`. Or upgrade borgmatic!)
+
+## Logging
+
+By default, borgmatic logs to a local syslog-compatible daemon if one is
+present and borgmatic is running in a non-interactive console. Where those
+logs show up depends on your particular system. If you're using systemd, try
+running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or
+similiar.
+
+You can customize the log level used for syslog logging with the
+`--syslog-verbosity` flag, and this is independent from the console logging
+`--verbosity` flag described above. For instance, to get additional
+information about the progress of the backup as it proceeds:
+
+```bash
+borgmatic --syslog-verbosity 1
+```
+
+Or to increase syslog logging to include debug spew:
+
+```bash
+borgmatic --syslog-verbosity 2
+```
+
+### systemd journal
+
+If your local syslog daemon is systemd's journal, be aware that journald by
+default throttles the rate at which a particular program can log. So you may
+need to [change the journald rate
+limit](https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=)
+in `/etc/systemd/journald.conf` if you're finding that borgmatic journald logs
+are missing.
+
+Note that the [sample borgmatic systemd service
+file](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#systemd)
+already has this rate limit disabled.
+
 ## Scripting borgmatic
 
 To consume the output of borgmatic in other software, you can include an
-optional `--json` flag with `--create`, `--list`, or `--info` to get the
-output formatted as JSON.
+optional `--json` flag with `create`, `list`, or `info` to get the output
+formatted as JSON.
+
+Note that when you specify the `--json` flag, Borg's other non-JSON output is
+suppressed so as not to interfere with the captured JSON. Also note that JSON
+output only shows up at the console, and not in syslog.
 
 
 ## Related documentation
 
- * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
- * [Develop on borgmatic](../../docs/how-to/develop-on-borgmatic.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)

docs/how-to/make-per-application-backups.md

@@ -105,7 +105,11 @@ include, the local file's option takes precedent. And note that this is a
 shallow merge rather than a deep merge, so the merging does not descend into
 nested values.
 
+Note that this `<<` include merging syntax is only for merging in mappings
+(keys/values). If you'd like to include other types like scalars or lists
+directly, please see the section above about standard includes.
+
 
 ## Related documentation
 
- * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)

docs/how-to/restore-a-backup.md

@@ -5,12 +5,15 @@ title: How to restore a backup
 
 When the worst happens—or you want to test your backups—the first step is
 to figure out which archive to restore. A good way to do that is to use the
-`--list` action:
+`list` action:
 
 ```bash
-borgmatic --list
+borgmatic list
 ```
 
+(No borgmatic `list` action? Try the old-style `--list`, or upgrade
+borgmatic!)
+
 That should yield output looking something like:
 
 ```text
@@ -22,9 +25,12 @@ Assuming that you want to restore the archive with the most up-to-date files
 and therefore the latest timestamp, run a command like:
 
 ```bash
-borgmatic --extract --archive host-2019-01-02T04:06:07.080910
+borgmatic extract --archive host-2019-01-02T04:06:07.080910
 ```
 
+(No borgmatic `extract` action? Try the old-style `--extract`, or upgrade
+borgmatic!)
+
 The `--archive` value is the name of the archive to restore. This extracts the
 entire contents of the archive to the current directory, so make sure you're
 in the right place before running the command.
@@ -33,13 +39,13 @@ in the right place before running the command.
 ## Repository selection
 
 If you have a single repository in your borgmatic configuration file(s), no
-problem: the `--extract` action figures out which repository to use.
+problem: the `extract` action figures out which repository to use.
 
 But if you have multiple repositories configured, then you'll need to specify
 the repository path containing the archive to extract. Here's an example:
 
 ```bash
-borgmatic --extract --repository repo.borg --archive host-2019-...
+borgmatic extract --repository repo.borg --archive host-2019-...
 ```
 
 ## Restore particular files
@@ -49,7 +55,7 @@ everything from an archive. To do that, tack on one or more `--restore-path`
 values. For instance:
 
 ```bash
-borgmatic --extract --archive host-2019-... --restore-path /path/1 /path/2
+borgmatic extract --archive host-2019-... --restore-path /path/1 /path/2
 ```
 
 Like a whole-archive restore, this also restores into the current directory.
@@ -57,5 +63,5 @@ Like a whole-archive restore, this also restores into the current directory.
 
 ## Related documentation
 
- * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
- * [Inspect your backups](../../docs/how-to/inspect-your-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)

docs/how-to/run-preparation-steps-before-backups.md

@@ -36,6 +36,12 @@ hooks:
         - echo "Error while creating a backup."
 ```
 
+## Hook output
+
+Any output produced by your hooks shows up both at the console and in syslog
+(when run in a non-interactive console). For more information, read about <a
+href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md">inspecting
+your backups</a>.
 
 ## Security
 
@@ -47,5 +53,6 @@ injection or privilege escalation, do not forget to set secure permissions
 
 ## Related documentation
 
- * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
- * [Make per-application backups](../../docs/how-to/make-per-application-backups.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)

docs/how-to/set-up-backups.md

@@ -7,16 +7,24 @@ To get up and running, first [install
 Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
 least version 1.1.
 
-Then, run the following command to download and install borgmatic:
+Borgmatic consumes configurations in `/etc/borgmatic/` and `/etc/borgmatic.d/`
+by default. Therefore, we show how to install borgmatic for the root user which
+will have access permissions for these locations by default.
+
+Run the following commands to download and install borgmatic:
 
 ```bash
-sudo pip3 install --upgrade borgmatic
+sudo pip3 install --user --upgrade borgmatic
 ```
 
+This is a [recommended user site
+installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site).
+You will need to ensure that `/root/.local/bin` is available on your `$PATH` so
+that the borgmatic executable is available.
+
 Note that your pip binary may have a different name than "pip3". Make sure
 you're using Python 3, as borgmatic does not support Python 2.
 
-
 ### Other ways to install
 
 Along with the above process, you have several other options for installing
@@ -32,6 +40,18 @@ borgmatic:
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
 
 
+## Hosting providers
+
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
+
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
+
 ## Configuration
 
 After you install borgmatic, generate a sample configuration file:
@@ -54,7 +74,7 @@ uncomment a particular option, also uncomment its containing section name, or
 else borgmatic won't recognize the option.
 
 You can also get the same sample configuration file from the [configuration
-reference](../../docs/reference/configuration.md), the authoritative set of
+reference](https://torsion.org/borgmatic/docs/reference/configuration.md), the authoritative set of
 all configuration options. This is handy if borgmatic has added new options
 since you originally created your configuration file.
 
@@ -76,7 +96,21 @@ FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-e
 for more info.
 
 
-###
+### Validation
+
+If you'd like to validate that your borgmatic configuration is valid, the
+following command is available for that:
+
+```bash
+sudo validate-borgmatic-config
+```
+
+This command's exit status (`$?` in Bash) is zero when configuration is valid
+and non-zero otherwise.
+
+Validating configuration can be useful if you generate your configuration
+files via configuration management, or you want to double check that your hand
+edits are valid.
 
 
 ## Initialization
@@ -87,9 +121,12 @@ this step if you already have a Borg repository.) To create a repository, run
 a command like the following:
 
 ```bash
-borgmatic --init --encryption repokey
+borgmatic init --encryption repokey
 ```
 
+(No borgmatic `init` action? Try the old-style `--init` flag, or upgrade
+borgmatic!)
+
 This uses the borgmatic configuration file you created above to determine
 which local or remote repository to create, and encrypts it with the
 encryption passphrase specified there if one is provided. Read about [Borg
@@ -151,9 +188,9 @@ You can modify the cron file if you'd like to run borgmatic more or less frequen
 
 If you're using systemd instead of cron to run jobs, download the [sample
 systemd service
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.service)
+file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
 and the [sample systemd timer
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.timer).
+file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
 Then, from the directory where you downloaded them:
 
 ```bash
@@ -165,6 +202,13 @@ sudo systemctl start borgmatic.timer
 Feel free to modify the timer file based on how frequently you'd like
 borgmatic to run.
 
+## Colored output
+
+Borgmatic produces colored terminal output by default. It is disabled when a
+non-interactive terminal is detected (like a cron job). Otherwise, you can
+disable it by passing the `--no-color` flag, setting the environment variable
+`PY_COLORS=False`, or setting the `color` option to `false` in the `output`
+section of configuration.
 
 ## Troubleshooting
 
@@ -181,8 +225,13 @@ it.
 
 ## Related documentation
 
- * [Make per-application backups](../../docs/how-to/make-per-application-backups.md)
- * [Deal with very large backups](../../docs/how-to/deal-with-very-large-backups.md)
- * [Inspect your backups](../../docs/how-to/inspect-your-backups.md)
- * [borgmatic configuration reference](../../docs/reference/configuration.md)
- * [borgmatic command-line reference](../../docs/reference/command-line.md)
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration.md)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line.md)
+
+<script>
+  var links = document.getElementsByClassName("referral");
+  links[Math.floor(Math.random() * links.length)].style.display = "none";
+</script>

docs/how-to/upgrade.md

@@ -7,7 +7,7 @@ In general, all you should need to do to upgrade borgmatic is run the
 following:
 
 ```bash
-sudo pip3 install --upgrade borgmatic
+sudo pip3 install --user --upgrade borgmatic
 ```
 
 See below about special cases.
@@ -25,14 +25,14 @@ already running borgmatic with Python 3, then you can upgrade borgmatic
 in-place:
 
 ```bash
-sudo pip3 install --upgrade borgmatic
+sudo pip3 install --user --upgrade borgmatic
 ```
 
 But if you were running borgmatic with Python 2, uninstall and reinstall instead:
 
 ```bash
 sudo pip uninstall borgmatic
-sudo pip3 install borgmatic
+sudo pip3 install --user borgmatic
 ```
 
 The pip binary names for different versions of Python can differ, so the above
@@ -67,7 +67,7 @@ from atticmatic to borgmatic by running the following commands:
 
 ```bash
 sudo pip3 uninstall atticmatic
-sudo pip3 install borgmatic
+sudo pip3 install --user borgmatic
 ```
 
 That's it! borgmatic will continue using your /etc/borgmatic configuration
@@ -76,4 +76,4 @@ files.
 
 ## Related documentation
 
- * [Develop on borgmatic](../../docs/how-to/develop-on-borgmatic.md)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)

docs/reference/command-line.md

@@ -3,7 +3,8 @@ title: borgmatic command-line reference
 ---
 ## borgmatic options
 
-Here are all of the available borgmatic command-line options:
+Here are all of the available borgmatic command-line options. This includes the separate options for
+each action sub-command:
 
 ```
 {% include borgmatic/command-line.txt %}
@@ -12,5 +13,5 @@ Here are all of the available borgmatic command-line options:
 
 ## Related documentation
 
- * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
- * [borgmatic configuration reference](../../docs/reference/configuration.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration.md)

docs/reference/configuration.md

@@ -10,10 +10,10 @@ Here is a full sample borgmatic configuration file including all available optio
 ```
 
 Note that you can also [download this configuration
-file](../../docs/reference/config.yaml) for use locally.
+file](https://torsion.org/borgmatic/docs/reference/config.yaml) for use locally.
 
 
 ## Related documentation
 
- * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
- * [borgmatic command-line reference](../../docs/reference/command-line.md)
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line.md)

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 100
+skip-string-normalization = true

sample/cron-alpine/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * PATH=$PATH:/usr/bin /usr/bin/borgmatic
+0 3 * * * PATH=$PATH:/usr/bin /root/.local/bin/borgmatic

sample/cron/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/borgmatic
+0 3 * * * root PATH=$PATH:/usr/local/bin /root/.local/bin/borgmatic

sample/systemd/borgmatic.service

@@ -3,4 +3,5 @@ Description=borgmatic backup
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/borgmatic
+ExecStart=/root/.local/bin/borgmatic
+LogRateLimitIntervalSec=0

scripts/black

@@ -1,7 +0,0 @@
-#!/usr/bin/env sh
-
-if which black; then
-    black --skip-string-normalization --line-length 100 --check .
-else
-    echo "Skipping black due to not being installed."
-fi

scripts/dev-docs

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+docker build --tag borgmatic-docs --file docs/Dockerfile .
+echo
+echo "You can view dev docs at http://localhost:8080"
+echo "Note that links within these docs will go to the online docs, so you will need to fiddle with URLs manually to stay in the dev docs."
+echo
+docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs

scripts/find-unsupported-borg-options

@@ -18,7 +18,7 @@ mv temp.yaml.uncommented temp.yaml
 for sub_command in prune create check list info; do
     echo "********** borg $sub_command **********"
 
-    for line in $(borgmatic --config temp.yaml --$sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
+    for line in $(borgmatic --config temp.yaml $sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
         echo "$line" | grep '^-' >> borgmatic_borg_flags
     done
     sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
@@ -48,6 +48,17 @@ for sub_command in prune create check list info; do
             | grep -v '^--stats$' \
             | grep -v '^--verbose$' \
             | grep -v '^--warning$' \
+            | grep -v '^--exclude' \
+            | grep -v '^--exclude-from' \
+            | grep -v '^--first' \
+            | grep -v '^--format' \
+            | grep -v '^--glob-archives' \
+            | grep -v '^--last' \
+            | grep -v '^--list-format' \
+            | grep -v '^--patterns-from' \
+            | grep -v '^--prefix' \
+            | grep -v '^--short' \
+            | grep -v '^--sort-by' \
             | grep -v '^-h$' \
             >> all_borg_flags
     done

scripts/pip

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+# Temporary work around for https://github.com/pypa/pip/issues/6434
+python -m pip install --upgrade pip==19.1.1
+python -m pip install --no-use-pep517 $*
+

scripts/run-tests

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# This script is intended to be run from the continuous integration build
+# server, and not on a developer machine. For that, see:
+# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
+
+set -e
+
+python -m pip install --upgrade pip==19.1.1
+pip install tox==3.10.0
+tox
+apk add --no-cache borgbackup
+tox -e end-to-end

setup.cfg

@@ -1,2 +1,20 @@
 [metadata]
 description-file=README.md
+
+[tool:pytest]
+testpaths = tests
+addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end
+filterwarnings = 
+    ignore:Coverage disabled.*:pytest.PytestWarning
+
+[flake8]
+ignore = E501,W503
+exclude = *.*/*
+
+[tool:isort]
+force_single_line = False
+include_trailing_comma = True
+known_first_party = borgmatic
+line_length = 100
+multi_line_output = 3
+skip = .tox

setup.py

@@ -1,13 +1,12 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-
-VERSION = '1.3.2'
+VERSION = '1.3.12'
 
 
 setup(
     name='borgmatic',
     version=VERSION,
-    description='A wrapper script for Borg backup software that creates and prunes backups',
+    description='Simple, configuration-driven backup software for servers and workstations',
     author='Dan Helfman',
     author_email='witten@torsion.org',
     url='https://torsion.org/borgmatic',
@@ -26,9 +25,15 @@ setup(
             'borgmatic = borgmatic.commands.borgmatic:main',
             'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
             'generate-borgmatic-config = borgmatic.commands.generate_config:main',
+            'validate-borgmatic-config = borgmatic.commands.validate_config:main',
         ]
     },
     obsoletes=['atticmatic'],
-    install_requires=('pykwalify>=1.6.0,<14.06', 'ruamel.yaml>0.15.0,<0.16.0', 'setuptools'),
+    install_requires=(
+        'pykwalify>=1.6.0,<14.06',
+        'ruamel.yaml>0.15.0,<0.16.0',
+        'setuptools',
+        'colorama>=0.4.1,<0.5',
+    ),
     include_package_data=True,
 )

test_requirements.txt

@@ -1,23 +1,24 @@
 appdirs==1.4.3
-atomicwrites==1.2.1
-attrs==18.2.0
-black==18.9b0; python_version >= '3.6'
-Click==7.0
-coverage==4.5.1
+atomicwrites==1.3.0
+attrs==19.1.0
+black==19.3b0; python_version >= '3.6'
+click==7.0
+colorama==0.4.1
+coverage==4.5.3
 docopt==0.6.2
-flake8==3.5.0
-flexmock==0.10.2
+flake8==3.7.7
+flexmock==0.10.4
+isort==4.3.20
 mccabe==0.6.1
-more-itertools==4.3.0
-pluggy==0.7.1
-py==1.6.0
-pycodestyle==2.3.1
-pyflakes==2.0.0
+more-itertools==7.0.0
+pluggy==0.12.0
+py==1.8.0
+pycodestyle==2.5.0
+pyflakes==2.1.1
 pykwalify==1.7.0
-pytest==3.8.2
-pytest-cov==2.6.0
-python-dateutil==2.7.3
-PyYAML==3.13
+pytest==4.6.3
+pytest-cov==2.7.1
+python-dateutil==2.8.0
+PyYAML==5.1.1
 ruamel.yaml>0.15.0,<0.16.0
-six==1.11.0
 toml==0.10.0

tests/end-to-end/test_validate_config.py

@@ -0,0 +1,36 @@
+import os
+import subprocess
+import tempfile
+
+
+def test_validate_config_command_with_valid_configuration_succeeds():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 0
+
+
+def test_validate_config_command_with_invalid_configuration_fails():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"')
+        config_file = open(config_path, 'w')
+        config_file.write(config)
+        config_file.close()
+
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 1

tests/integration/commands/test_arguments.py

@@ -0,0 +1,348 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.commands import arguments as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments()
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == ['myconfig', 'otherconfig']
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--verbosity', '1')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 1
+    assert global_arguments.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_syslog_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--syslog-verbosity', '2')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 2
+
+
+def test_parse_arguments_with_list_json_overrides_default():
+    arguments = module.parse_arguments('list', '--json')
+
+    assert 'list' in arguments
+    assert arguments['list'].json is True
+
+
+def test_parse_arguments_with_dashed_list_json_overrides_default():
+    arguments = module.parse_arguments('--list', '--json')
+
+    assert 'list' in arguments
+    assert arguments['list'].json is True
+
+
+def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments()
+
+    assert 'prune' in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_help_and_no_actions_shows_global_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'global arguments:' in captured.out
+    assert 'actions:' in captured.out
+
+
+def test_parse_arguments_with_help_and_action_shows_action_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('create', '--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'global arguments:' not in captured.out
+    assert 'actions:' not in captured.out
+    assert 'create arguments:' in captured.out
+
+
+def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('prune')
+
+    assert 'prune' in arguments
+    assert 'create' not in arguments
+    assert 'check' not in arguments
+
+
+def test_parse_arguments_with_dashed_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--prune')
+
+    assert 'prune' in arguments
+    assert 'create' not in arguments
+    assert 'check' not in arguments
+
+
+def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('create', 'check')
+
+    assert 'prune' not in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_multiple_dashed_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--create', '--check')
+
+    assert 'prune' not in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')
+
+
+def test_parse_arguments_disallows_deprecated_excludes_option():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+
+def test_parse_arguments_disallows_encryption_mode_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
+
+
+def test_parse_arguments_allows_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey')
+
+
+def test_parse_arguments_allows_encryption_mode_with_dashed_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey')
+
+
+def test_parse_arguments_requires_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', 'init')
+
+
+def test_parse_arguments_disallows_append_only_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--append-only')
+
+
+def test_parse_arguments_disallows_storage_quota_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
+
+
+def test_parse_arguments_allows_init_and_prune():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune')
+
+
+def test_parse_arguments_allows_init_and_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create')
+
+
+def test_parse_arguments_disallows_init_and_dry_run():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments(
+            '--config', 'myconfig', 'init', '--encryption', 'repokey', '--dry-run'
+        )
+
+
+def test_parse_arguments_disallows_repository_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
+
+
+def test_parse_arguments_allows_repository_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments(
+        '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test'
+    )
+
+
+def test_parse_arguments_allows_repository_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg')
+
+
+def test_parse_arguments_disallows_archive_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--archive', 'test')
+
+
+def test_parse_arguments_disallows_restore_paths_without_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--restore-path', 'test')
+
+
+def test_parse_arguments_allows_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_dashed_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'list', '--archive', 'test')
+
+
+def test_parse_arguments_requires_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', 'extract')
+
+
+def test_parse_arguments_allows_progress_before_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', 'create', 'list')
+
+
+def test_parse_arguments_allows_progress_after_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('create', '--progress', 'list')
+
+
+def test_parse_arguments_allows_progress_and_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list')
+
+
+def test_parse_arguments_disallows_progress_without_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--progress', 'list')
+
+
+def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', 'create', 'list')
+
+
+def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', 'prune', 'list')
+
+
+def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--stats', 'list')
+
+
+def test_parse_arguments_with_just_stats_flag_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats')
+
+
+def test_parse_arguments_allows_json_with_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('list', '--json')
+    module.parse_arguments('info', '--json')
+
+
+def test_parse_arguments_allows_json_with_dashed_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--info', '--json')
+
+
+def test_parse_arguments_disallows_json_with_both_list_and_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('list', 'info', '--json')

tests/integration/commands/test_borgmatic.py

@@ -1,266 +1,10 @@
 import subprocess
 
 from flexmock import flexmock
-import pytest
 
 from borgmatic.commands import borgmatic as module
 
 
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments()
-
-    assert parser.config_paths == config_paths
-    assert parser.excludes_filename is None
-    assert parser.verbosity is 0
-    assert parser.json is False
-
-
-def test_parse_arguments_with_multiple_config_paths_parses_as_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
-
-    assert parser.config_paths == ['myconfig', 'otherconfig']
-    assert parser.verbosity is 0
-
-
-def test_parse_arguments_with_verbosity_overrides_default():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments('--verbosity', '1')
-
-    assert parser.config_paths == config_paths
-    assert parser.excludes_filename is None
-    assert parser.verbosity == 1
-
-
-def test_parse_arguments_with_json_overrides_default():
-    parser = module.parse_arguments('--list', '--json')
-    assert parser.json is True
-
-
-def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments()
-
-    assert parser.prune is True
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--prune')
-
-    assert parser.prune is True
-    assert parser.create is False
-    assert parser.check is False
-
-
-def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--create', '--check')
-
-    assert parser.prune is False
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_invalid_arguments_exits():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(SystemExit):
-        module.parse_arguments('--posix-me-harder')
-
-
-def test_parse_arguments_disallows_deprecated_excludes_option():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
-
-
-def test_parse_arguments_disallows_encryption_mode_without_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
-
-
-def test_parse_arguments_allows_encryption_mode_with_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey')
-
-
-def test_parse_arguments_requires_encryption_mode_with_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--init')
-
-
-def test_parse_arguments_disallows_append_only_without_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--append-only')
-
-
-def test_parse_arguments_disallows_storage_quota_without_init():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
-
-
-def test_parse_arguments_allows_init_and_prune():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--prune')
-
-
-def test_parse_arguments_allows_init_and_create():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--create')
-
-
-def test_parse_arguments_disallows_init_and_dry_run():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments(
-            '--config', 'myconfig', '--init', '--encryption', 'repokey', '--dry-run'
-        )
-
-
-def test_parse_arguments_disallows_repository_without_extract_or_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
-
-
-def test_parse_arguments_allows_repository_with_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments(
-        '--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
-    )
-
-
-def test_parse_arguments_allows_repository_with_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
-
-
-def test_parse_arguments_disallows_archive_without_extract_or_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--archive', 'test')
-
-
-def test_parse_arguments_disallows_restore_paths_without_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--restore-path', 'test')
-
-
-def test_parse_arguments_allows_archive_with_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
-
-
-def test_parse_arguments_allows_archive_with_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
-
-
-def test_parse_arguments_requires_archive_with_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--config', 'myconfig', '--extract')
-
-
-def test_parse_arguments_allows_progress_and_create():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--progress', '--create', '--list')
-
-
-def test_parse_arguments_allows_progress_and_extract():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
-
-
-def test_parse_arguments_disallows_progress_without_create():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--progress', '--list')
-
-
-def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--stats', '--create', '--list')
-
-
-def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--stats', '--prune', '--list')
-
-
-def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--stats', '--list')
-
-
-def test_parse_arguments_with_just_stats_flag_does_not_raise():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--stats')
-
-
-def test_parse_arguments_allows_json_with_list_or_info():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--list', '--json')
-    module.parse_arguments('--info', '--json')
-
-
-def test_parse_arguments_disallows_json_without_list_or_info():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--json')
-
-
-def test_parse_arguments_disallows_json_with_both_list_and_info():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments('--list', '--info', '--json')
-
-
 def test_borgmatic_version_matches_news_version():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/commands/test_convert_config.py

@@ -1,7 +1,7 @@
 import os
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.commands import convert_config as module
 

tests/integration/commands/test_validate_config.py

@@ -0,0 +1,20 @@
+from flexmock import flexmock
+
+from borgmatic.commands import validate_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_paths == config_paths
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    assert parser.config_paths == ['myconfig', 'otherconfig']

tests/integration/config/test_generate.py

@@ -1,9 +1,9 @@
-from io import StringIO
 import os
 import sys
+from io import StringIO
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import generate as module
 

tests/integration/config/test_legacy.py

@@ -1,7 +1,6 @@
-from io import StringIO
-
-from collections import OrderedDict
 import string
+from collections import OrderedDict
+from io import StringIO
 
 from borgmatic.config import legacy as module
 

tests/integration/config/test_load.py

@@ -1,5 +1,7 @@
 import sys
 
+import pytest
+import ruamel.yaml
 from flexmock import flexmock
 
 from borgmatic.config import load as module
@@ -38,3 +40,23 @@ def test_load_configuration_merges_include():
     )
 
     assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
+
+
+def test_load_configuration_does_not_merge_include_list():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+          - one
+          - two
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: bar
+        repositories:
+          <<: !include include.yaml
+        '''
+    )
+
+    with pytest.raises(ruamel.yaml.error.YAMLError):
+        assert module.load_configuration('config.yaml')

tests/integration/config/test_validate.py

@@ -2,8 +2,8 @@ import io
 import string
 import sys
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import validate as module
 

tests/integration/test_execute.py

@@ -0,0 +1,46 @@
+import logging
+import subprocess
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic import execute as module
+
+
+def test_execute_and_log_output_logs_each_line_separately():
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
+
+    module.execute_and_log_output(['echo', 'hi'], output_log_level=logging.INFO, shell=False)
+    module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False)
+
+
+def test_execute_and_log_output_includes_borg_error_output_in_exception():
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+
+    assert error.value.returncode == 2
+    assert error.value.output
+
+
+def test_execute_and_log_output_truncates_long_borg_error_output():
+    flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+
+    assert error.value.returncode == 2
+    assert error.value.output.startswith('...')
+
+
+def test_execute_and_log_output_with_no_output_logs_nothing():
+    flexmock(module.logger).should_receive('log').never()
+
+    module.execute_and_log_output(['true'], output_log_level=logging.INFO, shell=False)
+
+
+def test_execute_and_log_output_with_error_exit_status_raises():
+    flexmock(module.logger).should_receive('log').never()
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)

tests/unit/borg/test_check.py

@@ -1,22 +1,19 @@
-from subprocess import STDOUT
 import logging
-import sys
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.borg import check as module
+
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+def insert_execute_command_mock(command):
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
-def insert_subprocess_never():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
+def insert_execute_command_never():
+    flexmock(module).should_receive('execute_command').never()
 
 
 def test_parse_checks_returns_them_as_tuple():
@@ -119,10 +116,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
     flexmock(module).should_receive('_make_check_flags').with_args(
         checks, check_last, None
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(('borg', 'check', 'repo'), stdout=stdout, stderr=STDOUT)
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -136,7 +130,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').never()
     flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
-    insert_subprocess_never()
+    insert_execute_command_never()
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -149,7 +143,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_logging_mock(logging.INFO)
-    insert_subprocess_mock(('borg', 'check', 'repo', '--info'), stdout=None, stderr=STDOUT)
+    insert_execute_command_mock(('borg', 'check', 'repo', '--info'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -162,9 +156,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_logging_mock(logging.DEBUG)
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--debug', '--show-rc'), stdout=None, stderr=STDOUT
-    )
+    insert_execute_command_mock(('borg', 'check', 'repo', '--debug', '--show-rc'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -174,7 +166,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
 def test_check_archives_without_any_checks_bails():
     consistency_config = {'check_last': None}
     flexmock(module).should_receive('_parse_checks').and_return(())
-    insert_subprocess_never()
+    insert_execute_command_never()
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -189,10 +181,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('_make_check_flags').with_args(
         checks, check_last, None
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(('borg1', 'check', 'repo'), stdout=stdout, stderr=STDOUT)
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg1', 'check', 'repo'))
 
     module.check_archives(
         repository='repo',
@@ -210,12 +199,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('_make_check_flags').with_args(
         checks, check_last, None
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--remote-path', 'borg1'), stdout=stdout, stderr=STDOUT
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', 'repo', '--remote-path', 'borg1'))
 
     module.check_archives(
         repository='repo',
@@ -233,12 +217,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('_make_check_flags').with_args(
         checks, check_last, None
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--lock-wait', '5'), stdout=stdout, stderr=STDOUT
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', 'repo', '--lock-wait', '5'))
 
     module.check_archives(
         repository='repo', storage_config={'lock_wait': 5}, consistency_config=consistency_config
@@ -254,11 +233,7 @@ def test_check_archives_with_retention_prefix():
     flexmock(module).should_receive('_make_check_flags').with_args(
         checks, check_last, prefix
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(('borg', 'check', 'repo'), stdout=stdout, stderr=STDOUT)
-
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config

tests/unit/borg/test_create.py

@@ -1,8 +1,10 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import create as module
+
 from ..test_verbosity import insert_logging_mock
 
 
@@ -165,7 +167,7 @@ def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND, capture_output=False
+        CREATE_COMMAND, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -190,7 +192,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
     flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + pattern_flags, capture_output=False
+        CREATE_COMMAND + pattern_flags, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -215,7 +217,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + exclude_flags, capture_output=False
+        CREATE_COMMAND + exclude_flags, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -238,7 +240,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--stats'), capture_output=False
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--stats'),
+        output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.INFO)
 
@@ -254,6 +257,31 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     )
 
 
+def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--json',), output_log_level=None
+    )
+    insert_logging_mock(logging.INFO)
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        json=True,
+    )
+
+
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
@@ -262,7 +290,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
         CREATE_COMMAND + ('--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc'),
-        capture_output=False,
+        output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -278,6 +306,30 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     )
 
 
+def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--json',), output_log_level=None
+    )
+    insert_logging_mock(logging.DEBUG)
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        json=True,
+    )
+
+
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
@@ -286,7 +338,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--dry-run',), capture_output=False
+        CREATE_COMMAND + ('--dry-run',), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -311,7 +363,8 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--dry-run'), capture_output=False
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--dry-run'),
+        output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.INFO)
 
@@ -338,7 +391,7 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
         CREATE_COMMAND + ('--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run'),
-        capture_output=False,
+        output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -361,7 +414,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--checkpoint-interval', '600'), capture_output=False
+        CREATE_COMMAND + ('--checkpoint-interval', '600'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -383,7 +436,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--chunker-params', '1,2,3,4'), capture_output=False
+        CREATE_COMMAND + ('--chunker-params', '1,2,3,4'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -405,7 +458,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--compression', 'rle'), capture_output=False
+        CREATE_COMMAND + ('--compression', 'rle'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -427,7 +480,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--remote-ratelimit', '100'), capture_output=False
+        CREATE_COMMAND + ('--remote-ratelimit', '100'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -449,7 +502,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--one-file-system',), capture_output=False
+        CREATE_COMMAND + ('--one-file-system',), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -472,7 +525,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--numeric-owner',), capture_output=False
+        CREATE_COMMAND + ('--numeric-owner',), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -495,7 +548,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--read-special',), capture_output=False
+        CREATE_COMMAND + ('--read-special',), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -511,14 +564,15 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     )
 
 
-def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter():
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND, capture_output=False
+        CREATE_COMMAND, output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -527,21 +581,22 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': True,
+            option_name: True,
             'exclude_patterns': None,
         },
         storage_config={},
     )
 
 
-def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter():
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--nobsdflags',), capture_output=False
+        CREATE_COMMAND + ('--no' + option_name.replace('_', ''),), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -550,7 +605,7 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': False,
+            option_name: False,
             'exclude_patterns': None,
         },
         storage_config={},
@@ -564,7 +619,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--files-cache', 'ctime,size'), capture_output=False
+        CREATE_COMMAND + ('--files-cache', 'ctime,size'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -587,7 +642,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1',) + CREATE_COMMAND[1:], capture_output=False
+        ('borg1',) + CREATE_COMMAND[1:], output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -610,7 +665,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+        CREATE_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -633,7 +688,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--umask', '740'), capture_output=False
+        CREATE_COMMAND + ('--umask', '740'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -655,7 +710,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--lock-wait', '5'), capture_output=False
+        CREATE_COMMAND + ('--lock-wait', '5'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -670,6 +725,29 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     )
 
 
+def test_create_archive_with_stats_calls_borg_with_stats_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--stats',), output_log_level=logging.WARNING
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        stats=True,
+    )
+
+
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
@@ -677,7 +755,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--json',), capture_output=True
+        CREATE_COMMAND + ('--json',), output_log_level=None
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -695,6 +773,32 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     assert json_output == '[]'
 
 
+def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--json',), output_log_level=None
+    ).and_return('[]')
+
+    json_output = module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        json=True,
+        stats=True,
+    )
+
+    assert json_output == '[]'
+
+
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
@@ -703,7 +807,7 @@ def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
-        capture_output=False,
+        output_log_level=logging.INFO,
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -726,7 +830,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'), capture_output=False
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'),
+        output_log_level=logging.INFO,
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
@@ -750,7 +855,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
-        capture_output=False,
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -772,7 +877,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'), capture_output=False
+        ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -794,7 +899,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'), capture_output=False
+        ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'),
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(

tests/unit/borg/test_environment.py

@@ -42,8 +42,7 @@ def test_initialize_without_configuration_should_not_set_environment():
     try:
         os.environ = {}
         module.initialize({})
-        assert os.environ.get('BORG_PASSCOMMAND') is None
-        assert os.environ.get('BORG_PASSPHRASE') is None
-        assert os.environ.get('BORG_RSH') is None
+
+        assert sum(1 for key in os.environ.keys() if key.startswith('BORG_')) == 0
     finally:
         os.environ = orig_environ

tests/unit/borg/test_execute.py

@@ -1,25 +0,0 @@
-from flexmock import flexmock
-from borgmatic.borg import execute as module
-
-
-def test_execute_command_calls_full_command():
-    full_command = ['foo', 'bar']
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(full_command).once()
-
-    output = module.execute_command(full_command)
-
-    assert output is None
-
-
-def test_execute_command_captures_output():
-    full_command = ['foo', 'bar']
-    expected_output = '[]'
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(full_command).and_return(
-        flexmock(decode=lambda: expected_output)
-    ).once()
-
-    output = module.execute_command(full_command, capture_output=True)
-
-    assert output == expected_output

tests/unit/borg/test_extract.py

@@ -1,67 +1,52 @@
 import logging
-import sys
 
 from flexmock import flexmock
 
 from borgmatic.borg import extract as module
+
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+def insert_execute_command_mock(command):
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
-def insert_subprocess_never():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
-
-
-def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(
-        result
-    ).once()
+def insert_execute_command_output_mock(command, result):
+    flexmock(module).should_receive('execute_command').with_args(
+        command, output_log_level=None
+    ).and_return(result).once()
 
 
 def test_extract_last_archive_dry_run_calls_borg_with_last_archive():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo'), result='archive1\narchive2\n'.encode('utf-8')
+    insert_execute_command_output_mock(
+        ('borg', 'list', '--short', 'repo'), result='archive1\narchive2\n'
     )
-    insert_subprocess_mock(('borg', 'extract', '--dry-run', 'repo::archive2'))
+    insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive2'))
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
 
 
-def test_extract_last_archive_dry_run_without_any_archives_should_bail():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo'), result='\n'.encode('utf-8')
-    )
-    insert_subprocess_never()
+def test_extract_last_archive_dry_run_without_any_archives_should_not_raise():
+    insert_execute_command_output_mock(('borg', 'list', '--short', 'repo'), result='\n')
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
 
 
 def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_parameter():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--info'), result='archive1\narchive2\n'.encode('utf-8')
+    insert_execute_command_output_mock(
+        ('borg', 'list', '--short', 'repo', '--info'), result='archive1\narchive2\n'
     )
-    insert_subprocess_mock(('borg', 'extract', '--dry-run', 'repo::archive2', '--info'))
+    insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive2', '--info'))
     insert_logging_mock(logging.INFO)
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
 
 
 def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_parameter():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--debug', '--show-rc'),
-        result='archive1\narchive2\n'.encode('utf-8'),
+    insert_execute_command_output_mock(
+        ('borg', 'list', '--short', 'repo', '--debug', '--show-rc'), result='archive1\narchive2\n'
     )
-    insert_subprocess_mock(
+    insert_execute_command_mock(
         ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--show-rc', '--list')
     )
     insert_logging_mock(logging.DEBUG)
@@ -70,22 +55,19 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param
 
 
 def test_extract_last_archive_dry_run_calls_borg_via_local_path():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg1', 'list', '--short', 'repo'), result='archive1\narchive2\n'.encode('utf-8')
+    insert_execute_command_output_mock(
+        ('borg1', 'list', '--short', 'repo'), result='archive1\narchive2\n'
     )
-    insert_subprocess_mock(('borg1', 'extract', '--dry-run', 'repo::archive2'))
+    insert_execute_command_mock(('borg1', 'extract', '--dry-run', 'repo::archive2'))
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None, local_path='borg1')
 
 
 def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
-        result='archive1\narchive2\n'.encode('utf-8'),
+    insert_execute_command_output_mock(
+        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'), result='archive1\narchive2\n'
     )
-    insert_subprocess_mock(
+    insert_execute_command_mock(
         ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1')
     )
 
@@ -93,18 +75,18 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters():
 
 
 def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
-        result='archive1\narchive2\n'.encode('utf-8'),
+    insert_execute_command_output_mock(
+        ('borg', 'list', '--short', 'repo', '--lock-wait', '5'), result='archive1\narchive2\n'
+    )
+    insert_execute_command_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5')
     )
-    insert_subprocess_mock(('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'))
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=5)
 
 
 def test_extract_archive_calls_borg_with_restore_path_parameters():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', 'path1', 'path2'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', 'path1', 'path2'))
 
     module.extract_archive(
         dry_run=False,
@@ -117,7 +99,7 @@ def test_extract_archive_calls_borg_with_restore_path_parameters():
 
 
 def test_extract_archive_calls_borg_with_remote_path_parameters():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--remote-path', 'borg1'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--remote-path', 'borg1'))
 
     module.extract_archive(
         dry_run=False,
@@ -131,7 +113,7 @@ def test_extract_archive_calls_borg_with_remote_path_parameters():
 
 
 def test_extract_archive_calls_borg_with_numeric_owner_parameter():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--numeric-owner'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--numeric-owner'))
 
     module.extract_archive(
         dry_run=False,
@@ -144,7 +126,7 @@ def test_extract_archive_calls_borg_with_numeric_owner_parameter():
 
 
 def test_extract_archive_calls_borg_with_umask_parameters():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--umask', '0770'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--umask', '0770'))
 
     module.extract_archive(
         dry_run=False,
@@ -157,7 +139,7 @@ def test_extract_archive_calls_borg_with_umask_parameters():
 
 
 def test_extract_archive_calls_borg_with_lock_wait_parameters():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--lock-wait', '5'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--lock-wait', '5'))
 
     module.extract_archive(
         dry_run=False,
@@ -170,7 +152,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters():
 
 
 def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--info'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--info'))
     insert_logging_mock(logging.INFO)
 
     module.extract_archive(
@@ -184,7 +166,9 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
 
 
 def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--debug', '--list', '--show-rc'))
+    insert_execute_command_mock(
+        ('borg', 'extract', 'repo::archive', '--debug', '--list', '--show-rc')
+    )
     insert_logging_mock(logging.DEBUG)
 
     module.extract_archive(
@@ -198,7 +182,7 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
 
 
 def test_extract_archive_calls_borg_with_dry_run_parameter():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--dry-run'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--dry-run'))
 
     module.extract_archive(
         dry_run=True,
@@ -211,7 +195,7 @@ def test_extract_archive_calls_borg_with_dry_run_parameter():
 
 
 def test_extract_archive_calls_borg_with_progress_parameter():
-    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--progress'))
+    insert_execute_command_mock(('borg', 'extract', 'repo::archive', '--progress'))
 
     module.extract_archive(
         dry_run=False,

tests/unit/borg/test_flags.py

@@ -0,0 +1,47 @@
+from flexmock import flexmock
+
+from borgmatic.borg import flags as module
+
+
+def test_make_flags_formats_string_value():
+    assert module.make_flags('foo', 'bar') == ('--foo', 'bar')
+
+
+def test_make_flags_formats_integer_value():
+    assert module.make_flags('foo', 3) == ('--foo', '3')
+
+
+def test_make_flags_formats_true_value():
+    assert module.make_flags('foo', True) == ('--foo',)
+
+
+def test_make_flags_omits_false_value():
+    assert module.make_flags('foo', False) == ()
+
+
+def test_make_flags_formats_name_with_underscore():
+    assert module.make_flags('posix_me_harder', 'okay') == ('--posix-me-harder', 'okay')
+
+
+def test_make_flags_from_arguments_flattens_and_sorts_multiple_arguments():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('make_flags').with_args('baz', 'quux').and_return(
+        ('baz', 'quux')
+    )
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('baz', 'quux', 'foo', 'bar')
+
+
+def test_make_flags_from_arguments_excludes_underscored_argument_names():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', _baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('foo', 'bar')
+
+
+def test_make_flags_from_arguments_omits_excludes():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments, excludes=('baz', 'other')) == ('foo', 'bar')

tests/unit/borg/test_info.py

@@ -1,67 +1,142 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import info as module
-from ..test_verbosity import insert_logging_mock
 
+from ..test_verbosity import insert_logging_mock
 
 INFO_COMMAND = ('borg', 'info', 'repo')
 
 
 def test_display_archives_info_calls_borg_with_parameters():
-    flexmock(module).should_receive('execute_command').with_args(INFO_COMMAND, capture_output=False)
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND, output_log_level=logging.WARNING
+    )
 
-    module.display_archives_info(repository='repo', storage_config={})
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--info',), capture_output=False
+        INFO_COMMAND + ('--info',), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.INFO)
-    module.display_archives_info(repository='repo', storage_config={})
-
-
-def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
-    flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--debug', '--show-rc'), capture_output=False
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
     )
-    insert_logging_mock(logging.DEBUG)
-
-    module.display_archives_info(repository='repo', storage_config={})
 
 
-def test_display_archives_info_with_json_calls_borg_with_json_parameter():
+def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--json',), capture_output=True
+        INFO_COMMAND + ('--json',), output_log_level=None
     ).and_return('[]')
 
-    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+    insert_logging_mock(logging.INFO)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'
 
 
-def test_display_archives_info_with_local_path_calls_borg_via_local_path():
+def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1',) + INFO_COMMAND[1:], capture_output=False
+        INFO_COMMAND + ('--debug', '--show-rc'), output_log_level=logging.WARNING
+    )
+    insert_logging_mock(logging.DEBUG)
+
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
     )
 
-    module.display_archives_info(repository='repo', storage_config={}, local_path='borg1')
+
+def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--json',), output_log_level=None
+    ).and_return('[]')
+
+    insert_logging_mock(logging.DEBUG)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
+
+    assert json_output == '[]'
+
+
+def test_display_archives_info_with_json_calls_borg_with_json_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--json',), output_log_level=None
+    ).and_return('[]')
+
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
+
+    assert json_output == '[]'
+
+
+def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING
+    )
+
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive='archive', json=False)
+    )
+
+
+def test_display_archives_info_with_local_path_calls_borg_via_local_path():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1',) + INFO_COMMAND[1:], output_log_level=logging.WARNING
+    )
+
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False),
+        local_path='borg1',
+    )
 
 
 def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+        INFO_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config={}, remote_path='borg1')
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False),
+        remote_path='borg1',
+    )
 
 
 def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
     flexmock(module).should_receive('execute_command').with_args(
-        INFO_COMMAND + ('--lock-wait', '5'), capture_output=False
+        INFO_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config=storage_config)
+    module.display_archives_info(
+        repository='repo',
+        storage_config=storage_config,
+        info_arguments=flexmock(archive=None, json=False),
+    )
+
+
+@pytest.mark.parametrize('argument_name', ('prefix', 'glob_archives', 'sort_by', 'first', 'last'))
+def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--' + argument_name.replace('_', '-'), 'value'),
+        output_log_level=logging.WARNING,
+    )
+
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}),
+    )

tests/unit/borg/test_init.py

@@ -1,56 +1,72 @@
 import logging
+import subprocess
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import init as module
+
 from ..test_verbosity import insert_logging_mock
 
-
-INFO_REPOSITORY_EXISTS_RESPONSE_CODE = 0
-INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE = 2
+INFO_SOME_UNKNOWN_EXIT_CODE = -999
 INIT_COMMAND = ('borg', 'init', 'repo', '--encryption', 'repokey')
 
 
-def insert_info_command_mock(info_response):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('call').and_return(info_response)
+def insert_info_command_found_mock():
+    flexmock(module).should_receive('execute_command')
+
+
+def insert_info_command_not_found_mock():
+    flexmock(module).should_receive('execute_command').and_raise(
+        subprocess.CalledProcessError(module.INFO_REPOSITORY_NOT_FOUND_EXIT_CODE, [])
+    )
 
 
 def insert_init_command_mock(init_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(init_command, **kwargs).once()
+    flexmock(module.subprocess).should_receive('check_call').with_args(
+        init_command, **kwargs
+    ).once()
 
 
 def test_initialize_repository_calls_borg_with_parameters():
-    insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
+    insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND)
 
     module.initialize_repository(repository='repo', encryption_mode='repokey')
 
 
 def test_initialize_repository_skips_initialization_when_repository_already_exists():
-    insert_info_command_mock(INFO_REPOSITORY_EXISTS_RESPONSE_CODE)
+    insert_info_command_found_mock()
     flexmock(module.subprocess).should_receive('check_call').never()
 
     module.initialize_repository(repository='repo', encryption_mode='repokey')
 
 
+def test_initialize_repository_raises_for_unknown_info_command_error():
+    flexmock(module).should_receive('execute_command').and_raise(
+        subprocess.CalledProcessError(INFO_SOME_UNKNOWN_EXIT_CODE, [])
+    )
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.initialize_repository(repository='repo', encryption_mode='repokey')
+
+
 def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter():
-    insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
+    insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--append-only',))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', append_only=True)
 
 
 def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter():
-    insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
+    insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G'))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', storage_quota='5G')
 
 
 def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
-    insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
+    insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--info',))
     insert_logging_mock(logging.INFO)
 
@@ -58,7 +74,7 @@ def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
 
 
 def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
-    insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
+    insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--debug',))
     insert_logging_mock(logging.DEBUG)
 
@@ -66,14 +82,14 @@ def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
 
 
 def test_initialize_repository_with_local_path_calls_borg_via_local_path():
-    insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
+    insert_info_command_not_found_mock()
     insert_init_command_mock(('borg1',) + INIT_COMMAND[1:])
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', local_path='borg1')
 
 
 def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter():
-    insert_info_command_mock(INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE)
+    insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1'))
 
     module.initialize_repository(repository='repo', encryption_mode='repokey', remote_path='borg1')

tests/unit/borg/test_list.py

@@ -1,77 +1,167 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import list as module
-from ..test_verbosity import insert_logging_mock
 
+from ..test_verbosity import insert_logging_mock
 
 LIST_COMMAND = ('borg', 'list', 'repo')
 
 
 def test_list_archives_calls_borg_with_parameters():
-    flexmock(module).should_receive('execute_command').with_args(LIST_COMMAND, capture_output=False)
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND, output_log_level=logging.WARNING
+    )
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--info',), capture_output=False
+        LIST_COMMAND + ('--info',), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
+
+
+def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--json',), output_log_level=None
+    )
+    insert_logging_mock(logging.INFO)
+
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
 
 def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--debug', '--show-rc'), capture_output=False
+        LIST_COMMAND + ('--debug', '--show-rc'), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
+
+
+def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--json',), output_log_level=None
+    )
+    insert_logging_mock(logging.DEBUG)
+
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
 
 def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--lock-wait', '5'), capture_output=False
+        LIST_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config)
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive=None, json=False),
+    )
 
 
 def test_list_archives_with_archive_calls_borg_with_archive_parameter():
     storage_config = {}
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo::archive'), capture_output=False
+        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive='archive', json=False),
+    )
 
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1',) + LIST_COMMAND[1:], capture_output=False
+        ('borg1',) + LIST_COMMAND[1:], output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, local_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False),
+        local_path='borg1',
+    )
 
 
 def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+        LIST_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False),
+        remote_path='borg1',
+    )
+
+
+def test_list_archives_with_short_calls_borg_with_short_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--short',), output_log_level=logging.WARNING
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, short=True),
+    )
+
+
+@pytest.mark.parametrize(
+    'argument_name',
+    (
+        'prefix',
+        'glob_archives',
+        'sort_by',
+        'first',
+        'last',
+        'exclude',
+        'exclude_from',
+        'pattern',
+        'pattern_from',
+    ),
+)
+def test_list_archives_passes_through_arguments_to_borg(argument_name):
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--' + argument_name.replace('_', '-'), 'value'),
+        output_log_level=logging.WARNING,
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}),
+    )
 
 
 def test_list_archives_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        LIST_COMMAND + ('--json',), capture_output=True
+        LIST_COMMAND + ('--json',), output_log_level=None
     ).and_return('[]')
 
-    json_output = module.list_archives(repository='repo', storage_config={}, json=True)
+    json_output = module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'

tests/unit/borg/test_prune.py

@@ -4,12 +4,12 @@ from collections import OrderedDict
 from flexmock import flexmock
 
 from borgmatic.borg import prune as module
+
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+def insert_execute_command_mock(prune_command, **kwargs):
+    flexmock(module).should_receive('execute_command').with_args(prune_command).once()
 
 
 BASE_PRUNE_FLAGS = (('--keep-daily', '1'), ('--keep-weekly', '2'), ('--keep-monthly', '3'))
@@ -51,7 +51,7 @@ def test_prune_archives_calls_borg_with_parameters():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(PRUNE_COMMAND)
+    insert_execute_command_mock(PRUNE_COMMAND)
 
     module.prune_archives(
         dry_run=False, repository='repo', storage_config={}, retention_config=retention_config
@@ -63,7 +63,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--stats', '--info'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--stats', '--info'))
     insert_logging_mock(logging.INFO)
 
     module.prune_archives(
@@ -76,7 +76,7 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--stats', '--debug', '--list', '--show-rc'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--stats', '--debug', '--list', '--show-rc'))
     insert_logging_mock(logging.DEBUG)
 
     module.prune_archives(
@@ -89,7 +89,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--dry-run',))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run',))
 
     module.prune_archives(
         repository='repo', storage_config={}, dry_run=True, retention_config=retention_config
@@ -101,7 +101,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(('borg1',) + PRUNE_COMMAND[1:])
+    insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:])
 
     module.prune_archives(
         dry_run=False,
@@ -117,7 +117,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
 
     module.prune_archives(
         dry_run=False,
@@ -134,7 +134,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--umask', '077'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077'))
 
     module.prune_archives(
         dry_run=False,
@@ -150,7 +150,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
+    insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
 
     module.prune_archives(
         dry_run=False,

tests/unit/commands/test_arguments.py

@@ -0,0 +1,171 @@
+from flexmock import flexmock
+
+from borgmatic.commands import arguments as module
+
+
+def test_parse_subparser_arguments_consumes_subparser_arguments_before_subparser_name():
+    global_namespace = flexmock()
+    action_namespace = flexmock(foo=True)
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(
+        ('--foo', 'true', 'action'), top_level_parser, subparsers
+    )
+
+    assert arguments == {'action': action_namespace, 'global': global_namespace}
+
+
+def test_parse_subparser_arguments_consumes_subparser_arguments_after_subparser_name():
+    global_namespace = flexmock()
+    action_namespace = flexmock(foo=True)
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(
+        ('action', '--foo', 'true'), top_level_parser, subparsers
+    )
+
+    assert arguments == {'action': action_namespace, 'global': global_namespace}
+
+
+def test_parse_subparser_arguments_consumes_global_arguments_before_subparser_name():
+    global_namespace = flexmock(verbosity='lots')
+    action_namespace = flexmock()
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
+            ),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(
+        ('--verbosity', 'lots', 'action'), top_level_parser, subparsers
+    )
+
+    assert arguments == {'action': action_namespace, 'global': global_namespace}
+
+
+def test_parse_subparser_arguments_consumes_global_arguments_after_subparser_name():
+    global_namespace = flexmock(verbosity='lots')
+    action_namespace = flexmock()
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
+            ),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(
+        ('action', '--verbosity', 'lots'), top_level_parser, subparsers
+    )
+
+    assert arguments == {'action': action_namespace, 'global': global_namespace}
+
+
+def test_parse_subparser_arguments_consumes_subparser_arguments_with_alias():
+    global_namespace = flexmock()
+    action_namespace = flexmock(foo=True)
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, []))
+    subparsers = flexmock(
+        choices={
+            'action': action_subparser,
+            '-a': action_subparser,
+            'other': flexmock(),
+            '-o': flexmock(),
+        }
+    )
+    flexmock(module).SUBPARSER_ALIASES = {'action': ['-a'], 'other': ['-o']}
+
+    arguments = module.parse_subparser_arguments(
+        ('-a', '--foo', 'true'), top_level_parser, subparsers
+    )
+
+    assert arguments == {'action': action_namespace, 'global': global_namespace}
+
+
+def test_parse_subparser_arguments_consumes_multiple_subparser_arguments():
+    global_namespace = flexmock()
+    action_namespace = flexmock(foo=True)
+    other_namespace = flexmock(bar=3)
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--bar', '3'])
+            ),
+            'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(
+        ('action', '--foo', 'true', 'other', '--bar', '3'), top_level_parser, subparsers
+    )
+
+    assert arguments == {
+        'action': action_namespace,
+        'other': other_namespace,
+        'global': global_namespace,
+    }
+
+
+def test_parse_subparser_arguments_applies_default_subparsers():
+    global_namespace = flexmock()
+    prune_namespace = flexmock()
+    create_namespace = flexmock(progress=True)
+    check_namespace = flexmock()
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'prune': flexmock(parse_known_args=lambda arguments: (prune_namespace, ['--progress'])),
+            'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])),
+            'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(('--progress'), top_level_parser, subparsers)
+
+    assert arguments == {
+        'prune': prune_namespace,
+        'create': create_namespace,
+        'check': check_namespace,
+        'global': global_namespace,
+    }
+
+
+def test_parse_subparser_arguments_with_help_does_not_apply_default_subparsers():
+    global_namespace = flexmock(verbosity='lots')
+    action_namespace = flexmock()
+    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
+    subparsers = flexmock(
+        choices={
+            'action': flexmock(
+                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
+            ),
+            'other': flexmock(),
+        }
+    )
+
+    arguments = module.parse_subparser_arguments(
+        ('--verbosity', 'lots', '--help'), top_level_parser, subparsers
+    )
+
+    assert arguments == {'global': global_namespace}

tests/unit/commands/test_borgmatic.py

@@ -1,101 +1,137 @@
+import subprocess
+
 from flexmock import flexmock
 
 from borgmatic.commands import borgmatic as module
 
 
-def test_collect_configuration_run_summary_logs_info_for_success():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
-    flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(extract=False, list=False)
+def test_load_configurations_collects_parsed_configurations():
+    configuration = flexmock()
+    other_configuration = flexmock()
+    flexmock(module.validate).should_receive('parse_configuration').and_return(
+        configuration
+    ).and_return(other_configuration)
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    configs, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml')))
+
+    assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration}
+    assert logs == []
+
+
+def test_load_configurations_logs_critical_for_parse_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
+
+    configs, logs = tuple(module.load_configurations(('test.yaml',)))
+
+    assert configs == {}
+    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+
+
+def test_collect_configuration_run_summary_logs_info_for_success():
+    flexmock(module).should_receive('run_configuration').and_return([])
+    arguments = {}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
     assert all(log for log in logs if log.levelno == module.logging.INFO)
 
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
     flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(extract=True, list=False, repository='repo')
+    arguments = {'extract': flexmock(repository='repo')}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
     assert all(log for log in logs if log.levelno == module.logging.INFO)
 
 
 def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
         ValueError
     )
-    args = flexmock(extract=True, list=False, repository='repo')
+    arguments = {'extract': flexmock(repository='repo')}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
 
 
 def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
         ValueError
     )
-    args = flexmock(extract=False, list=True, repository='repo', archive='test')
+    arguments = {'list': flexmock(repository='repo', archive='test')}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
 
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_list():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(extract=False, list=True, repository='repo', archive=None)
+    arguments = {'list': flexmock(repository='repo', archive=None)}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
     assert all(log for log in logs if log.levelno == module.logging.INFO)
 
 
-def test_collect_configuration_run_summary_logs_critical_for_parse_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
-    args = flexmock(extract=False, list=False)
-
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
-
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
-
-
-def test_collect_configuration_run_summary_logs_critical_for_run_error():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+def test_collect_configuration_run_summary_logs_critical_for_run_value_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
     flexmock(module).should_receive('run_configuration').and_raise(ValueError)
-    args = flexmock(extract=False, list=False)
+    arguments = {}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
 
 
-def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
-    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
-    flexmock(module).should_receive('run_configuration').and_return([])
-    args = flexmock(config_paths=(), extract=False, list=False)
+def test_collect_configuration_run_summary_logs_critical_including_output_for_run_process_error():
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository')
+    flexmock(module).should_receive('run_configuration').and_raise(
+        subprocess.CalledProcessError(1, 'command', 'error output')
+    )
+    arguments = {}
 
-    logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args))
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert any(log for log in logs if 'error output' in str(log))
 
 
 def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
-    flexmock(module.validate).should_receive('parse_configuration').and_return(
-        {'test.yaml': {}, 'test2.yaml': {}}
-    )
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
         ['baz']
     )
     flexmock(module.sys.stdout).should_receive('write').with_args('["foo", "bar", "baz"]').once()
-    args = flexmock(extract=False, list=False)
+    arguments = {}
 
-    tuple(module.collect_configuration_run_summary_logs(('test.yaml', 'test2.yaml'), args=args))
+    tuple(
+        module.collect_configuration_run_summary_logs(
+            {'test.yaml': {}, 'test2.yaml': {}}, arguments=arguments
+        )
+    )
+
+
+def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
+    flexmock(module).should_receive('run_configuration').and_return([])
+    arguments = {'global': flexmock(config_paths=[])}
+
+    logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
+
+    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)

tests/unit/commands/test_hook.py

@@ -1,29 +0,0 @@
-from flexmock import flexmock
-
-from borgmatic.commands import hook as module
-
-
-def test_execute_hook_invokes_each_command():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(':', shell=True).once()
-
-    module.execute_hook([':'], 'config.yaml', 'pre-backup', dry_run=False)
-
-
-def test_execute_hook_with_multiple_commands_invokes_each_command():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(':', shell=True).once()
-    subprocess.should_receive('check_call').with_args('true', shell=True).once()
-
-    module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup', dry_run=False)
-
-
-def test_execute_hook_with_dry_run_skips_commands():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
-
-    module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup', dry_run=True)
-
-
-def test_execute_hook_with_empty_commands_does_not_raise():
-    module.execute_hook([], 'config.yaml', 'post-backup', dry_run=False)

tests/unit/config/test_collect.py

@@ -30,6 +30,21 @@ def test_collect_config_filenames_collects_given_files():
     assert config_filenames == config_paths
 
 
+def test_collect_config_filenames_collects_yml_file_endings():
+    config_paths = ('config.yaml', '/etc/borgmatic.d')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').and_return(True)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False)
+    flexmock(module.os).should_receive('listdir')
+    flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml'])
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('config.yaml', '/etc/borgmatic.d/foo.yml')
+
+
 def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
     config_paths = ('config.yaml', '/etc/borgmatic.d')
     mock_path = flexmock(module.os.path)

tests/unit/config/test_convert.py

@@ -1,12 +1,11 @@
-from collections import defaultdict, OrderedDict, namedtuple
 import os
+from collections import OrderedDict, defaultdict, namedtuple
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import convert as module
 
-
 Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
 
 
@@ -91,7 +90,7 @@ def test_guard_configuration_upgraded_raises_when_only_source_config_present():
     flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
     flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
 
-    with pytest.raises(module.LegacyConfigurationNotUpgraded):
+    with pytest.raises(module.Legacy_configuration_not_upgraded):
         module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
 
 

tests/unit/config/test_legacy.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import legacy as module
 

tests/unit/config/test_validate.py

@@ -1,5 +1,4 @@
 import pytest
-from flexmock import flexmock
 
 from borgmatic.config import validate as module
 
@@ -37,20 +36,6 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
         )
 
 
-def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix():
-    logger = flexmock(module.logger)
-    logger.should_receive('warning').once()
-
-    module.apply_logical_validation(
-        'config.yaml',
-        {
-            'storage': {'archive_name_format': '{hostname}-{now}'},
-            'retention': {'prefix': '{hostname}-'},
-            'consistency': {},
-        },
-    )
-
-
 def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories():
     with pytest.raises(module.Validation_error):
         module.apply_logical_validation(
@@ -74,10 +59,7 @@ def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_re
     )
 
 
-def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
-    logger = flexmock(module.logger)
-    logger.should_receive('warning').never()
-
+def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
     module.apply_logical_validation(
         'config.yaml',
         {

tests/unit/test_execute.py

@@ -0,0 +1,51 @@
+import logging
+
+from flexmock import flexmock
+
+from borgmatic import execute as module
+
+
+def test_execute_command_calls_full_command():
+    full_command = ['foo', 'bar']
+    flexmock(module).should_receive('execute_and_log_output').with_args(
+        full_command, output_log_level=logging.INFO, shell=False
+    ).once()
+
+    output = module.execute_command(full_command)
+
+    assert output is None
+
+
+def test_execute_command_calls_full_command_with_shell():
+    full_command = ['foo', 'bar']
+    flexmock(module).should_receive('execute_and_log_output').with_args(
+        full_command, output_log_level=logging.INFO, shell=True
+    ).once()
+
+    output = module.execute_command(full_command, shell=True)
+
+    assert output is None
+
+
+def test_execute_command_captures_output():
+    full_command = ['foo', 'bar']
+    expected_output = '[]'
+    flexmock(module.subprocess).should_receive('check_output').with_args(
+        full_command, shell=False
+    ).and_return(flexmock(decode=lambda: expected_output)).once()
+
+    output = module.execute_command(full_command, output_log_level=None)
+
+    assert output == expected_output
+
+
+def test_execute_command_captures_output_with_shell():
+    full_command = ['foo', 'bar']
+    expected_output = '[]'
+    flexmock(module.subprocess).should_receive('check_output').with_args(
+        full_command, shell=True
+    ).and_return(flexmock(decode=lambda: expected_output)).once()
+
+    output = module.execute_command(full_command, output_log_level=None, shell=True)
+
+    assert output == expected_output

tests/unit/test_hook.py

@@ -0,0 +1,52 @@
+import logging
+
+from flexmock import flexmock
+
+from borgmatic import hook as module
+
+
+def test_execute_hook_invokes_each_command():
+    flexmock(module.execute).should_receive('execute_command').with_args(
+        [':'], output_log_level=logging.WARNING, shell=True
+    ).once()
+
+    module.execute_hook([':'], None, 'config.yaml', 'pre-backup', dry_run=False)
+
+
+def test_execute_hook_with_multiple_commands_invokes_each_command():
+    flexmock(module.execute).should_receive('execute_command').with_args(
+        [':'], output_log_level=logging.WARNING, shell=True
+    ).once()
+    flexmock(module.execute).should_receive('execute_command').with_args(
+        ['true'], output_log_level=logging.WARNING, shell=True
+    ).once()
+
+    module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=False)
+
+
+def test_execute_hook_with_umask_sets_that_umask():
+    flexmock(module.os).should_receive('umask').with_args(0o77).and_return(0o22).once()
+    flexmock(module.os).should_receive('umask').with_args(0o22).once()
+    flexmock(module.execute).should_receive('execute_command').with_args(
+        [':'], output_log_level=logging.WARNING, shell=True
+    )
+
+    module.execute_hook([':'], 77, 'config.yaml', 'pre-backup', dry_run=False)
+
+
+def test_execute_hook_with_dry_run_skips_commands():
+    flexmock(module.execute).should_receive('execute_command').never()
+
+    module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=True)
+
+
+def test_execute_hook_with_empty_commands_does_not_raise():
+    module.execute_hook([], None, 'config.yaml', 'post-backup', dry_run=False)
+
+
+def test_execute_hook_on_error_logs_as_error():
+    flexmock(module.execute).should_receive('execute_command').with_args(
+        [':'], output_log_level=logging.ERROR, shell=True
+    ).once()
+
+    module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)