Fix for regression with missing verbose log entries (#177).

Closes https://projects.torsion.org/witten/borgmatic/issues/169.

.drone.yml

@@ -3,7 +3,8 @@ pipeline:
     image: python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
     pull: true
     commands:
-      - pip install tox
+      - python -m pip install --upgrade pip==19.1.1
+      - pip install tox==3.10.0
       - tox
       - apk add --no-cache borgbackup
       - tox -e end-to-end
@@ -11,7 +12,7 @@ pipeline:
 matrix:
   ALPINE_VERSION:
     - 3.7
-    - 3.8
+    - 3.9
   PYTHON_VERSION:
     - 3.5
     - 3.6

AUTHORS

@@ -7,6 +7,6 @@ Johannes Feichtner: Support for user hooks
 Michele Lazzeri: Custom archive names
 Nick Whyte: Support prefix filtering for archive consistency checks
 newtonne: Read encryption password from external file
-Robin `ypid` Schneider: Support additional options of Borg
+Robin `ypid` Schneider: Support additional options of Borg and add validate-borgmatic-config command
 Scott Squires: Custom archive names
 Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option

NEWS

@@ -1,3 +1,57 @@
+1.3.5
+ * #153: Support for various Borg directory environment variables (BORG_CONFIG_DIR, BORG_CACHE_DIR,
+   etc.) via options in borgmatic's storage configuration.
+ * #177: Fix for regression with missing verbose log entries.
+
+1.3.4
+ * Part of #125: Color borgmatic (but not Borg) output when using an interactive terminal.
+ * #166: Run tests for all installed versions of Python.
+ * #168: Update README with continuous integration badge.
+ * #169: Automatically sort Python imports in code.
+ * Document installing borgmatic with pip install --user instead of a system Python install.
+ * Get more reproducible builds by pinning the versions of pip and tox used to run tests.
+ * Factor out build/test configuration from tox.ini file.
+
+1.3.3
+ * Add validate-borgmatic-config command, useful for validating borgmatic config generated by
+   configuration management or even edited by hand.
+
+1.3.2
+ * #160: Fix for hooks executing when using --dry-run. Now hooks are skipped during a dry run.
+
+1.3.1
+ * #155: Fix for invalid JSON output when using multiple borgmatic configuration files.
+ * #157: Fix for seemingly random filename ordering when running through a directory of
+   configuration files.
+ * Fix for empty JSON output when using --create --json.
+ * Now capturing Borg output only when --json flag is used. Previously, borgmatic delayed Borg
+   output even without the --json flag.
+
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
+1.2.18
+ * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
+   borgmatic's location section.
+
+1.2.17
+ * #140: List the files within an archive via --list --archive option.
+
+1.2.16
+ * #119: Include a sample borgmatic configuration file in the documentation.
+ * #123: Support for Borg archive restoration via borgmatic --extract command-line flag.
+ * Refactor documentation into multiple separate pages for clarity and findability.
+ * Organize options within command-line help into logical groups.
+ * Exclude tests from distribution packages.
+
+1.2.15
+ * #127: Remove date echo from schema example, as it's not a substitute for real logging.
+ * #132: Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to
+   confusing behavior.
+ * #136: Handle and format validation errors raised during argument parsing.
+ * #138: Allow use of --stats flag when --create or --prune flags are implied.
+
 1.2.14
  * #103: When generating sample configuration with generate-borgmatic-config, document the defaults
    for each option.
@@ -22,8 +76,8 @@
  * #108: Support for Borg create --progress via borgmatic command-line flag.
 
 1.2.10
- * #105: Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage
-   section.
+ * #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
+   storage section.
 
 1.2.9
  * #102: Fix for syntax error that occurred in Python 3.5 and below.

README.md

@@ -2,9 +2,11 @@
 title: borgmatic
 permalink: borgmatic/index.html
 ---
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg)</a>
+
 ## Overview
 
-<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" width="150px" style="float: right; padding-left: 1em;">
+<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
 
 borgmatic is a simple Python wrapper script for the
 [Borg](https://www.borgbackup.org/) backup software that initiates a backup,
@@ -54,337 +56,22 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 <script src="https://asciinema.org/a/203761.js" id="asciicast-203761" async></script>
 
 
-## Installation
+## How-to guides
 
-To get up and running, first [install
-Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
-least version 1.1.
+ * [Set up backups with borgmatic](docs/how-to/set-up-backups.md) ⬅ *Start here!*
+ * [Make per-application backups](docs/how-to/make-per-application-backups.md)
+ * [Deal with very large backups](docs/how-to/deal-with-very-large-backups.md)
+ * [Inspect your backups](docs/how-to/inspect-your-backups.md)
+ * [Restore a backup](docs/how-to/restore-a-backup.md)
+ * [Run preparation steps before backups](docs/how-to/run-preparation-steps-before-backups.md)
+ * [Upgrade borgmatic](docs/how-to/upgrade.md)
+ * [Develop on borgmatic](docs/how-to/develop-on-borgmatic.md)
 
-Then, run the following command to download and install borgmatic:
 
-```bash
-sudo pip3 install --upgrade borgmatic
-```
+## Reference guides
 
-Note that your pip binary may have a different name than "pip3". Make sure
-you're using Python 3, as borgmatic does not support Python 2.
-
-
-### Other ways to install
-
- * [Docker image](https://hub.docker.com/r/monachus/borgmatic/)
- * [Another Docker image](https://hub.docker.com/r/b3vis/borgmatic/)
- * [Debian](https://tracker.debian.org/pkg/borgmatic)
- * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
- * [Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
- * [Arch Linux](https://aur.archlinux.org/packages/borgmatic/)
- * [OpenBSD](http://ports.su/sysutils/borgmatic)
-<br><br>
-
-
-## Configuration
-
-After you install borgmatic, generate a sample configuration file:
-
-```bash
-sudo generate-borgmatic-config
-```
-
-If that command is not found, then it may be installed in a location that's
-not in your system `PATH`. Try looking in `/usr/local/bin/`.
-
-This generates a sample configuration file at /etc/borgmatic/config.yaml (by
-default). You should edit the file to suit your needs, as the values are just
-representative. All fields are optional except where indicated, so feel free
-to ignore anything you don't need.
-
-You can also have a look at the [full configuration
-schema](https://projects.torsion.org/witten/borgmatic/src/master/borgmatic/config/schema.yaml)
-for the authoritative set of all configuration options. This is handy if
-borgmatic has added new options since you originally created your
-configuration file.
-
-
-### Encryption
-
-Note that if you plan to run borgmatic on a schedule with cron, and you
-encrypt your Borg repository with a passphrase instead of a key file, you'll
-either need to set the borgmatic `encryption_passphrase` configuration
-variable or set the `BORG_PASSPHRASE` environment variable. See the
-[repository encryption
-section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
-of the Quick Start for more info.
-
-Alternatively, the passphrase can be specified programatically by setting
-either the borgmatic `encryption_passcommand` configuration variable or the
-`BORG_PASSCOMMAND` environment variable. See the [Borg Security
-FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
-for more info.
-
-
-## Usage
-
-### Initialization
-
-Before you can create backups with borgmatic, you first need to initialize a
-Borg repository so you have a destination for your backup archives. (But skip
-this step if you already have a Borg repository.) To create a repository, run
-a command like the following:
-
-```bash
-borgmatic --init --encryption repokey
-```
-
-This uses the borgmatic configuration file you created above to determine
-which local or remote repository to create, and encrypts it with the
-encryption passphrase specified there if one is provided. Read about [Borg
-encryption
-modes](https://borgbackup.readthedocs.io/en/latest/usage/init.html#encryption-modes)
-for the menu of available encryption modes.
-
-Also, optionally check out the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) for more
-background about repository initialization.
-
-Note that borgmatic skips repository initialization if the repository already
-exists. This supports use cases like ensuring a repository exists prior to
-performing a backup.
-
-If the repository is on a remote host, make sure that your local user has
-key-based SSH access to the desired user account on the remote host.
-
-
-### Backups
-
-You can run borgmatic and start a backup simply by invoking it without
-arguments:
-
-```bash
-borgmatic
-```
-
-This will also prune any old backups as per the configured retention policy,
-and check backups for consistency problems due to things like file damage.
-
-If you'd like to see the available command-line arguments, view the help:
-
-```bash
-borgmatic --help
-```
-
-Note that borgmatic prunes archives *before* creating an archive, so as to
-free up space for archiving. This means that when a borgmatic run finishes,
-there may still be prune-able archives. Not to worry, as they will get cleaned
-up at the start of the next run.
-
-
-### Verbosity
-
-By default, the backup will proceed silently except in the case of errors. But
-if you'd like to to get additional information about the progress of the
-backup as it proceeds, use the verbosity option:
-
-```bash
-borgmatic --verbosity 1
-```
-
-Or, for even more progress spew:
-
-```bash
-borgmatic --verbosity 2
-```
-
-### À la carte
-
-If you want to run borgmatic with only pruning, creating, or checking enabled,
-the following optional flags are available:
-
-```bash
-borgmatic --prune
-borgmatic --create
-borgmatic --check
-```
-
-You can run with only one of these flags provided, or you can mix and match
-any number of them. This supports use cases like running consistency checks
-from a different cron job with a different frequency, or running pruning with
-a different verbosity level.
-
-Additionally, borgmatic provides convenient flags for Borg's
-[list](https://borgbackup.readthedocs.io/en/stable/usage/list.html) and
-[info](https://borgbackup.readthedocs.io/en/stable/usage/info.html)
-functionality:
-
-
-```bash
-borgmatic --list
-borgmatic --info
-```
-
-You can include an optional `--json` flag with `--create`, `--list`, or
-`--info` to get the output formatted as JSON.
-
-
-## Autopilot
-
-If you want to run borgmatic automatically, say once a day, the you can
-configure a job runner to invoke it periodically.
-
-### cron
-
-If you're using cron, download the [sample cron
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic).
-Then, from the directory where you downloaded it:
-
-```bash
-sudo mv borgmatic /etc/cron.d/borgmatic
-sudo chmod +x /etc/cron.d/borgmatic
-```
-
-You can modify the cron file if you'd like to run borgmatic more or less frequently.
-
-### systemd
-
-If you're using systemd instead of cron to run jobs, download the [sample
-systemd service
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.service)
-and the [sample systemd timer
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.timer).
-Then, from the directory where you downloaded them:
-
-```bash
-sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
-sudo systemctl enable borgmatic.timer
-sudo systemctl start borgmatic.timer
-```
-
-Feel free to modify the timer file based on how frequently you'd like
-borgmatic to run.
-
-
-## Advanced configuration
-
-### Multiple configuration files
-
-A more advanced usage is to create multiple separate configuration files and
-place each one in an /etc/borgmatic.d directory. For instance:
-
-```bash
-sudo mkdir /etc/borgmatic.d
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
-```
-
-With this approach, you can have entirely different backup policies for
-different applications on your system. For instance, you may want one backup
-configuration for your database data directory, and a different configuration
-for your user home directories.
-
-When you set up multiple configuration files like this, borgmatic will run
-each one in turn from a single borgmatic invocation. This includes, by
-default, the traditional /etc/borgmatic/config.yaml as well.
-
-And if you need even more customizability, you can specify alternate
-configuration paths on the command-line with borgmatic's `--config` option.
-See `borgmatic --help` for more information.
-
-
-### Hooks
-
-If you find yourself performing prepraration tasks before your backup runs, or
-cleanup work afterwards, borgmatic hooks may be of interest. They're simply
-shell commands that borgmatic executes for you at various points, and they're
-configured in the `hooks` section of your configuration file.
-
-For instance, you can specify `before_backup` hooks to dump a database to file
-before backing it up, and specify `after_backup` hooks to delete the temporary
-file afterwards.
-
-borgmatic hooks run once per configuration file. `before_backup` hooks run
-prior to backups of all repositories. `after_backup` hooks run afterwards, but
-not if an error occurs in a previous hook or in the backups themselves. And
-borgmatic runs `on_error` hooks if an error occurs.
-
-An important security note about hooks: borgmatic executes all hook commands
-with the user permissions of borgmatic itself. So to prevent potential shell
-injection or privilege escalation, do not forget to set secure permissions
-(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
-
-See the sample generated configuration file mentioned above for specifics
-about hook configuration syntax.
-
-
-## Upgrading
-
-In general, all you should need to do to upgrade borgmatic is run the
-following:
-
-```bash
-sudo pip3 install --upgrade borgmatic
-```
-
-However, see below about special cases.
-
-
-### Upgrading from borgmatic 1.0.x
-
-borgmatic changed its configuration file format in version 1.1.0 from
-INI-style to YAML. This better supports validation, and has a more natural way
-to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic.
-
-As of version 1.1.0, borgmatic no longer supports Python 2. If you were
-already running borgmatic with Python 3, then you can simply upgrade borgmatic
-in-place:
-
-```bash
-sudo pip3 install --upgrade borgmatic
-```
-
-But if you were running borgmatic with Python 2, uninstall and reinstall instead:
-
-```bash
-sudo pip uninstall borgmatic
-sudo pip3 install borgmatic
-```
-
-The pip binary names for different versions of Python can differ, so the above
-commands may need some tweaking to work on your machine.
-
-
-Once borgmatic is upgraded, run:
-
-```bash
-sudo upgrade-borgmatic-config
-```
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and
-excludes files. The new version of borgmatic will consume the YAML
-configuration file instead of the old one.
-
-
-### Upgrading from atticmatic
-
-You can ignore this section if you're not an atticmatic user (the former name
-of borgmatic).
-
-borgmatic only supports Borg now and no longer supports Attic. So if you're
-an Attic user, consider switching to Borg. See the [Borg upgrade
-command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
-for more information. Then, follow the instructions above about setting up
-your borgmatic configuration files.
-
-If you were already using Borg with atticmatic, then you can easily upgrade
-from atticmatic to borgmatic. Simply run the following commands:
-
-```bash
-sudo pip3 uninstall atticmatic
-sudo pip3 install borgmatic
-```
-
-That's it! borgmatic will continue using your /etc/borgmatic configuration
-files.
+ * [borgmatic configuration reference](docs/reference/configuration.md)
+ * [borgmatic command-line reference](docs/reference/command-line.md)
 
 
 ## Support and contributing
@@ -408,127 +95,6 @@ or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
 to discuss your idea. We also accept Pull Requests on GitHub, if that's more
 your thing. In general, contributions are very welcome. We don't bite! 
 
-
-### Code style
-
-Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
-the following deviations from it:
-
- * For strings, prefer single quotes over double quotes.
- * Limit all lines to a maximum of 100 characters.
- * Use trailing commas within multiline values or argument lists.
- * For multiline constructs, put opening and closing delimeters on lines
-   separate from their contents.
- * Within multiline constructs, use standard four-space indentation. Don't align
-   indentation with an opening delimeter.
-
-borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
-formatter and [Flake8](http://flake8.pycqa.org/en/latest/) code checker, so
-certain code style requirements will be enforced when running automated tests.
-See the Black and Flake8 documentation for more information.
-
-
-### Development
-
-To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
-
-```bash
-git clone https://projects.torsion.org/witten/borgmatic.git
-```
-
-Or:
-
-```bash
-git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git
-```
-
-Then, install borgmatic
-"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)"
-so that you can easily run borgmatic commands while you're hacking on them to
-make sure your changes work.
-
-```bash
-cd borgmatic/
-pip3 install --editable --user .
-```
-
-Note that this will typically install the borgmatic commands into
-`~/.local/bin`, which may or may not be on your PATH. There are other ways to
-install borgmatic editable as well, for instance into the system Python
-install (so without `--user`, as root), or even into a
-[virtualenv](https://virtualenv.pypa.io/en/stable/). How or where you install
-borgmatic is up to you, but generally an editable install makes development
-and testing easier.
-
-
-### Running tests
-
-Assuming you've cloned the borgmatic source code as described above, and
-you're in the `borgmatic/` working copy, install tox, which is used for
-setting up testing environments:
-
-```bash
-sudo pip3 install tox
-```
-
-Finally, to actually run tests, run:
-
-```bash
-cd borgmatic
-tox
-```
-
-If when running tests, you get an error from the
-[Black](https://black.readthedocs.io/en/stable/) code formatter about files
-that would be reformatted, you can ask Black to format them for you via the
-following:
-
-```bash
-tox -e black
-```
-
-### End-to-end tests
-
-borgmatic additionally includes some end-to-end tests that integration test
-with Borg for a few representative scenarios. These tests don't run by default
-because they're relatively slow and depend on Borg. If you would like to run
-them:
-
-```bash
-tox -e end-to-end
-```
-
-## Troubleshooting
-
-### Broken pipe with remote repository
-
-When running borgmatic on a large remote repository, you may receive errors
-like the following, particularly while "borg check" is validating backups for
-consistency:
-
-```text
-    Write failed: Broken pipe
-    borg: Error: Connection closed by remote host
-```
-
-This error can be caused by an ssh timeout, which you can rectify by adding
-the following to the `~/.ssh/config` file on the client:
-
-```text
-    Host *
-        ServerAliveInterval 120
-```
-
-This should make the client keep the connection alive while validating
-backups.
-
-
-### libyaml compilation errors
-
-borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
-use a C YAML library (libyaml) if present. But if it's not installed, then
-when installing or upgrading borgmatic, you may see errors about compiling the
-YAML library. If so, not to worry. borgmatic should install and function
-correctly even without the C YAML library. And borgmatic won't be any faster
-with the C library present, so you don't need to go out of your way to install
-it.
+Also, please check out the [borgmatic development
+how-to](docs/how-to/develop-on-borgmatic.md) for info on cloning source code,
+running tests, etc.

borgmatic/borg/check.py

@@ -3,13 +3,13 @@ import os
 import subprocess
 
 from borgmatic.borg import extract
-
+from borgmatic.logger import get_logger
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
 
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 def _parse_checks(consistency_config):

borgmatic/borg/create.py

@@ -2,11 +2,12 @@ import glob
 import itertools
 import logging
 import os
-import subprocess
 import tempfile
 
+from borgmatic.borg.execute import execute_command
+from borgmatic.logger import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 def _expand_directory(directory):
@@ -32,6 +33,17 @@ def _expand_directories(directories):
     )
 
 
+def _expand_home_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
+    Return the results as a tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(os.path.expanduser(directory) for directory in directories)
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -96,12 +108,14 @@ def create_archive(
 ):
     '''
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
-    storage config dict, create a Borg archive.
+    storage config dict, create a Borg archive and return Borg's JSON output (if any).
     '''
     sources = _expand_directories(location_config['source_directories'])
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
-    exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
+    exclude_file = _write_pattern_file(
+        _expand_home_directories(location_config.get('exclude_patterns'))
+    )
     checkpoint_interval = storage_config.get('checkpoint_interval', None)
     chunker_params = storage_config.get('chunker_params', None)
     compression = storage_config.get('compression', None)
@@ -128,6 +142,7 @@ def create_archive(
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--read-special',) if location_config.get('read_special') else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
@@ -143,5 +158,4 @@ def create_archive(
         + (('--json',) if json else ())
     )
 
-    logger.debug(' '.join(full_command))
-    subprocess.check_call(full_command)
+    return execute_command(full_command, capture_output=json)

borgmatic/borg/environment.py

@@ -1,15 +1,19 @@
 import os
 
+OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'borg_base_directory': 'BORG_BASE_DIR',
+    'borg_config_directory': 'BORG_CONFIG_DIR',
+    'borg_cache_directory': 'BORG_CACHE_DIR',
+    'borg_security_directory': 'BORG_SECURITY_DIR',
+    'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
+    'ssh_command': 'BORG_RSH',
+}
+
 
 def initialize(storage_config):
-    passcommand = storage_config.get('encryption_passcommand')
-    if passcommand:
-        os.environ['BORG_PASSCOMMAND'] = passcommand
-
-    passphrase = storage_config.get('encryption_passphrase')
-    if passphrase:
-        os.environ['BORG_PASSPHRASE'] = passphrase
-
-    ssh_command = storage_config.get('ssh_command')
-    if ssh_command:
-        os.environ['BORG_RSH'] = ssh_command
+    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name)
+        if value:
+            os.environ[environment_variable_name] = value

borgmatic/borg/execute.py

@@ -0,0 +1,19 @@
+import subprocess
+
+from borgmatic.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def execute_command(full_command, capture_output=False):
+    '''
+    Execute the given command (a sequence of command/argument strings). If capture output is True,
+    then return the command's output as a string.
+    '''
+    logger.debug(' '.join(full_command))
+
+    if capture_output:
+        output = subprocess.check_output(full_command)
+        return output.decode() if output is not None else None
+    else:
+        subprocess.check_call(full_command)

borgmatic/borg/extract.py

@@ -1,15 +1,16 @@
 import logging
-import sys
 import subprocess
+import sys
 
+from borgmatic.logger import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg', remote_path=None):
     '''
-    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
-    the dry-run.
+    Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
+    dry-run.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
@@ -50,3 +51,39 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
 
     logger.debug(' '.join(full_extract_command))
     subprocess.check_call(full_extract_command)
+
+
+def extract_archive(
+    dry_run,
+    repository,
+    archive,
+    restore_paths,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
+    restore from the archive, and location/storage configuration dicts, extract the archive into the
+    current directory.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'extract', '::'.join((repository, archive)))
+        + (tuple(restore_paths) if restore_paths else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+    )
+
+    logger.debug(' '.join(full_command))
+    subprocess.check_call(full_command)

borgmatic/borg/info.py

@@ -1,16 +1,17 @@
 import logging
-import subprocess
 
+from borgmatic.borg.execute import execute_command
+from borgmatic.logger import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 def display_archives_info(
     repository, storage_config, local_path='borg', remote_path=None, json=False
 ):
     '''
-    Given a local or remote repository path, and a storage config dict,
-    display summary information for Borg archives in the repository.
+    Given a local or remote repository path, and a storage config dict, display summary information
+    for Borg archives in the repository or return JSON summary information.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
@@ -23,7 +24,4 @@ def display_archives_info(
         + (('--json',) if json else ())
     )
 
-    logger.debug(' '.join(full_command))
-
-    output = subprocess.check_output(full_command)
-    return output.decode() if output is not None else None
+    return execute_command(full_command, capture_output=json)

borgmatic/borg/init.py

@@ -1,8 +1,9 @@
 import logging
 import subprocess
 
+from borgmatic.logger import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 def initialize_repository(

borgmatic/borg/list.py

@@ -1,26 +1,28 @@
 import logging
-import subprocess
+
+from borgmatic.borg.execute import execute_command
+from borgmatic.logger import get_logger
+
+logger = get_logger(__name__)
 
 
-logger = logging.getLogger(__name__)
-
-
-def list_archives(repository, storage_config, local_path='borg', remote_path=None, json=False):
+def list_archives(
+    repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
+):
     '''
-    Given a local or remote repository path, and a storage config dict,
-    list Borg archives in the repository.
+    Given a local or remote repository path and a storage config dict, display the output of listing
+    Borg archives in the repository or return JSON output. Or, if an archive name is given, listing
+    the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'list', repository)
+        (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--json',) if json else ())
     )
-    logger.debug(' '.join(full_command))
 
-    output = subprocess.check_output(full_command)
-    return output.decode() if output is not None else None
+    return execute_command(full_command, capture_output=json)

borgmatic/borg/prune.py

@@ -1,8 +1,9 @@
 import logging
 import subprocess
 
+from borgmatic.logger import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 def _make_prune_flags(retention_config):

borgmatic/commands/borgmatic.py

@@ -1,29 +1,29 @@
-from argparse import ArgumentParser
+import collections
 import json
 import logging
 import os
-from subprocess import CalledProcessError
 import sys
+from argparse import ArgumentParser
+from subprocess import CalledProcessError
 
+import colorama
 import pkg_resources
 
-from borgmatic.borg import (
-    check as borg_check,
-    create as borg_create,
-    environment as borg_environment,
-    prune as borg_prune,
-    list as borg_list,
-    info as borg_info,
-    init as borg_init,
-)
+from borgmatic.borg import check as borg_check
+from borgmatic.borg import create as borg_create
+from borgmatic.borg import environment as borg_environment
+from borgmatic.borg import extract as borg_extract
+from borgmatic.borg import info as borg_info
+from borgmatic.borg import init as borg_init
+from borgmatic.borg import list as borg_list
+from borgmatic.borg import prune as borg_prune
 from borgmatic.commands import hook
 from borgmatic.config import checks, collect, convert, validate
+from borgmatic.logger import get_logger, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
 
-
-logger = logging.getLogger(__name__)
-
+logger = get_logger(__name__)
 
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
 
@@ -38,11 +38,114 @@ def parse_arguments(*arguments):
     parser = ArgumentParser(
         description='''
             A simple wrapper script for the Borg backup software that creates and prunes backups.
-            If none of the --prune, --create, or --check options are given, then borgmatic defaults
-            to all three: prune, create, and check archives.
-            '''
+            If none of the action options are given, then borgmatic defaults to: prune, create, and
+            check archives.
+            ''',
+        add_help=False,
     )
-    parser.add_argument(
+
+    actions_group = parser.add_argument_group('actions')
+    actions_group.add_argument(
+        '-I', '--init', dest='init', action='store_true', help='Initialize an empty Borg repository'
+    )
+    actions_group.add_argument(
+        '-p',
+        '--prune',
+        dest='prune',
+        action='store_true',
+        help='Prune archives according to the retention policy',
+    )
+    actions_group.add_argument(
+        '-C',
+        '--create',
+        dest='create',
+        action='store_true',
+        help='Create archives (actually perform backups)',
+    )
+    actions_group.add_argument(
+        '-k', '--check', dest='check', action='store_true', help='Check archives for consistency'
+    )
+
+    actions_group.add_argument(
+        '-x',
+        '--extract',
+        dest='extract',
+        action='store_true',
+        help='Extract a named archive to the current directory',
+    )
+    actions_group.add_argument(
+        '-l', '--list', dest='list', action='store_true', help='List archives'
+    )
+    actions_group.add_argument(
+        '-i',
+        '--info',
+        dest='info',
+        action='store_true',
+        help='Display summary information on archives',
+    )
+
+    init_group = parser.add_argument_group('options for --init')
+    init_group.add_argument(
+        '-e', '--encryption', dest='encryption_mode', help='Borg repository encryption mode'
+    )
+    init_group.add_argument(
+        '--append-only',
+        dest='append_only',
+        action='store_true',
+        help='Create an append-only repository',
+    )
+    init_group.add_argument(
+        '--storage-quota',
+        dest='storage_quota',
+        help='Create a repository with a fixed storage quota',
+    )
+
+    prune_group = parser.add_argument_group('options for --prune')
+    stats_argument = prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+
+    create_group = parser.add_argument_group('options for --create')
+    progress_argument = create_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    create_group._group_actions.append(stats_argument)
+    json_argument = create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+
+    extract_group = parser.add_argument_group('options for --extract')
+    repository_argument = extract_group.add_argument(
+        '--repository',
+        help='Path of repository to use, defaults to the configured repository if there is only one',
+    )
+    archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
+    extract_group.add_argument(
+        '--restore-path',
+        nargs='+',
+        dest='restore_paths',
+        help='Paths to restore from archive, defaults to the entire archive',
+    )
+    extract_group._group_actions.append(progress_argument)
+
+    list_group = parser.add_argument_group('options for --list')
+    list_group._group_actions.append(repository_argument)
+    list_group._group_actions.append(archive_argument)
+    list_group._group_actions.append(json_argument)
+
+    info_group = parser.add_argument_group('options for --info')
+    info_group._group_actions.append(json_argument)
+
+    common_group = parser.add_argument_group('common options')
+    common_group.add_argument(
         '-c',
         '--config',
         nargs='+',
@@ -52,85 +155,22 @@ def parse_arguments(*arguments):
             ' '.join(config_paths)
         ),
     )
-    parser.add_argument(
+    common_group.add_argument(
         '--excludes',
         dest='excludes_filename',
         help='Deprecated in favor of exclude_patterns within configuration',
     )
-    parser.add_argument(
-        '-I', '--init', dest='init', action='store_true', help='Initialize an empty Borg repository'
-    )
-    parser.add_argument(
-        '-e',
-        '--encryption',
-        dest='encryption_mode',
-        help='Borg repository encryption mode (for use with --init)',
-    )
-    parser.add_argument(
-        '--append-only',
-        dest='append_only',
-        action='store_true',
-        help='Create an append-only repository (for use with --init)',
-    )
-    parser.add_argument(
-        '--storage-quota',
-        dest='storage_quota',
-        help='Create a repository with a fixed storage quota (for use with --init)',
-    )
-    parser.add_argument(
-        '-p',
-        '--prune',
-        dest='prune',
-        action='store_true',
-        help='Prune archives according to the retention policy',
-    )
-    parser.add_argument(
-        '-C',
-        '--create',
-        dest='create',
-        action='store_true',
-        help='Create archives (actually perform backups)',
-    )
-    parser.add_argument(
-        '-k', '--check', dest='check', action='store_true', help='Check archives for consistency'
-    )
-    parser.add_argument('-l', '--list', dest='list', action='store_true', help='List archives')
-    parser.add_argument(
-        '-i',
-        '--info',
-        dest='info',
-        action='store_true',
-        help='Display summary information on archives',
-    )
-    parser.add_argument(
-        '--progress',
-        dest='progress',
-        default=False,
-        action='store_true',
-        help='Display progress with --create option for each file as it is backed up',
-    )
-    parser.add_argument(
-        '--stats',
-        dest='stats',
-        default=False,
-        action='store_true',
-        help='Display statistics of archive with --create or --prune option',
-    )
-    parser.add_argument(
-        '--json',
-        dest='json',
-        default=False,
-        action='store_true',
-        help='Output results from the --create, --list, or --info options as json',
-    )
-    parser.add_argument(
+    common_group.add_argument(
         '-n',
         '--dry-run',
         dest='dry_run',
         action='store_true',
         help='Go through the motions, but do not actually write to any repositories',
     )
-    parser.add_argument(
+    common_group.add_argument(
+        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
+    )
+    common_group.add_argument(
         '-v',
         '--verbosity',
         type=int,
@@ -138,13 +178,14 @@ def parse_arguments(*arguments):
         default=0,
         help='Display verbose progress (1 for some, 2 for lots)',
     )
-    parser.add_argument(
+    common_group.add_argument(
         '--version',
         dest='version',
         default=False,
         action='store_true',
         help='Display installed version number of borgmatic and exit',
     )
+    common_group.add_argument('--help', action='help', help='Show this help information and exit')
 
     args = parser.parse_args(arguments)
 
@@ -163,11 +204,25 @@ def parse_arguments(*arguments):
     if args.init and not args.encryption_mode:
         raise ValueError('The --encryption option is required with the --init option')
 
-    if args.progress and not args.create:
-        raise ValueError('The --progress option can only be used with the --create option')
+    if not args.extract:
+        if not args.list:
+            if args.repository:
+                raise ValueError(
+                    'The --repository option can only be used with the --extract and --list options'
+                )
+            if args.archive:
+                raise ValueError(
+                    'The --archive option can only be used with the --extract and --list options'
+                )
+        if args.restore_paths:
+            raise ValueError('The --restore-path option can only be used with the --extract option')
+    if args.extract and not args.archive:
+        raise ValueError('The --archive option is required with the --extract option')
 
-    if args.stats and not (args.create or args.prune):
-        raise ValueError('The --stats option can only be used with the --create or --prune options')
+    if args.progress and not (args.create or args.extract):
+        raise ValueError(
+            'The --progress option can only be used with the --create and --extract options'
+        )
 
     if args.json and not (args.create or args.list or args.info):
         raise ValueError(
@@ -181,22 +236,32 @@ def parse_arguments(*arguments):
 
     # If any of the action flags are explicitly requested, leave them as-is. Otherwise, assume
     # defaults: Mutate the given arguments to enable the default actions.
-    if args.init or args.prune or args.create or args.check or args.list or args.info:
-        return args
+    if (
+        not args.init
+        and not args.prune
+        and not args.create
+        and not args.check
+        and not args.extract
+        and not args.list
+        and not args.info
+    ):
+        args.prune = True
+        args.create = True
+        args.check = True
+
+    if args.stats and not (args.create or args.prune):
+        raise ValueError('The --stats option can only be used when creating or pruning archives')
 
-    args.prune = True
-    args.create = True
-    args.check = True
     return args
 
 
-def run_configuration(config_filename, args):  # pragma: no cover
+def run_configuration(config_filename, config, args):  # pragma: no cover
     '''
-    Parse a single configuration file, and execute its defined pruning, backups, and/or consistency
-    checks.
+    Given a config filename and the corresponding parsed config dict, execute its defined pruning,
+    backups, consistency checks, and/or other actions.
+
+    Yield JSON output strings from executing any actions that produce JSON.
     '''
-    logger.info('{}: Parsing configuration file'.format(config_filename))
-    config = validate.parse_configuration(config_filename, validate.schema_filename())
     (location, storage, retention, consistency, hooks) = (
         config.get(section_name, {})
         for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
@@ -208,56 +273,42 @@ def run_configuration(config_filename, args):  # pragma: no cover
         borg_environment.initialize(storage)
 
         if args.create:
-            hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
+            hook.execute_hook(
+                hooks.get('before_backup'), config_filename, 'pre-backup', args.dry_run
+            )
 
-        _run_commands(
-            args=args,
-            consistency=consistency,
-            local_path=local_path,
-            location=location,
-            remote_path=remote_path,
-            retention=retention,
-            storage=storage,
-        )
+        for repository_path in location['repositories']:
+            yield from run_actions(
+                args=args,
+                location=location,
+                storage=storage,
+                retention=retention,
+                consistency=consistency,
+                local_path=local_path,
+                remote_path=remote_path,
+                repository_path=repository_path,
+            )
 
         if args.create:
-            hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
+            hook.execute_hook(
+                hooks.get('after_backup'), config_filename, 'post-backup', args.dry_run
+            )
     except (OSError, CalledProcessError):
-        hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
+        hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error', args.dry_run)
         raise
 
 
-def _run_commands(*, args, consistency, local_path, location, remote_path, retention, storage):
-    json_results = []
-    for unexpanded_repository in location['repositories']:
-        _run_commands_on_repository(
-            args=args,
-            consistency=consistency,
-            json_results=json_results,
-            local_path=local_path,
-            location=location,
-            remote_path=remote_path,
-            retention=retention,
-            storage=storage,
-            unexpanded_repository=unexpanded_repository,
-        )
-    if args.json:
-        sys.stdout.write(json.dumps(json_results))
-
-
-def _run_commands_on_repository(
-    *,
-    args,
-    consistency,
-    json_results,
-    local_path,
-    location,
-    remote_path,
-    retention,
-    storage,
-    unexpanded_repository
+def run_actions(
+    *, args, location, storage, retention, consistency, local_path, remote_path, repository_path
 ):  # pragma: no cover
-    repository = os.path.expanduser(unexpanded_repository)
+    '''
+    Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
+    configuration dicts, local and remote paths to Borg, and a repository name, run all actions
+    from the command-line arguments on the given repository.
+
+    Yield JSON output strings from executing any actions that produce JSON.
+    '''
+    repository = os.path.expanduser(repository_path)
     dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
     if args.init:
         logger.info('{}: Initializing repository'.format(repository))
@@ -282,7 +333,7 @@ def _run_commands_on_repository(
         )
     if args.create:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
-        borg_create.create_archive(
+        json_output = borg_create.create_archive(
             args.dry_run,
             repository,
             location,
@@ -291,30 +342,49 @@ def _run_commands_on_repository(
             remote_path=remote_path,
             progress=args.progress,
             stats=args.stats,
+            json=args.json,
         )
+        if json_output:
+            yield json.loads(json_output)
     if args.check and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
             repository, storage, consistency, local_path=local_path, remote_path=remote_path
         )
+    if args.extract:
+        if args.repository is None or repository == args.repository:
+            logger.info('{}: Extracting archive {}'.format(repository, args.archive))
+            borg_extract.extract_archive(
+                args.dry_run,
+                repository,
+                args.archive,
+                args.restore_paths,
+                location,
+                storage,
+                local_path=local_path,
+                remote_path=remote_path,
+                progress=args.progress,
+            )
     if args.list:
-        logger.info('{}: Listing archives'.format(repository))
-        output = borg_list.list_archives(
-            repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
-        )
-        if args.json:
-            json_results.append(json.loads(output))
-        else:
-            sys.stdout.write(output)
+        if args.repository is None or repository == args.repository:
+            logger.info('{}: Listing archives'.format(repository))
+            json_output = borg_list.list_archives(
+                repository,
+                storage,
+                args.archive,
+                local_path=local_path,
+                remote_path=remote_path,
+                json=args.json,
+            )
+            if json_output:
+                yield json.loads(json_output)
     if args.info:
         logger.info('{}: Displaying summary info for archives'.format(repository))
-        output = borg_info.display_archives_info(
+        json_output = borg_info.display_archives_info(
             repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
         )
-        if args.json:
-            json_results.append(json.loads(output))
-        else:
-            sys.stdout.write(output)
+        if json_output:
+            yield json.loads(json_output)
 
 
 def collect_configuration_run_summary_logs(config_filenames, args):
@@ -322,10 +392,42 @@ def collect_configuration_run_summary_logs(config_filenames, args):
     Given a sequence of configuration filenames and parsed command-line arguments as an
     argparse.ArgumentParser instance, run each configuration file and yield a series of
     logging.LogRecord instances containing summary information about each run.
+
+    As a side effect of running through these configuration files, output their JSON results, if
+    any, to stdout.
     '''
+    # Dict mapping from config filename to corresponding parsed config dict.
+    configs = collections.OrderedDict()
+
+    # Parse and load each configuration file.
     for config_filename in config_filenames:
         try:
-            run_configuration(config_filename, args)
+            logger.info('{}: Parsing configuration file'.format(config_filename))
+            configs[config_filename] = validate.parse_configuration(
+                config_filename, validate.schema_filename()
+            )
+        except (ValueError, OSError, validate.Validation_error) as error:
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.CRITICAL,
+                    msg='{}: Error parsing configuration file'.format(config_filename),
+                )
+            )
+            yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
+
+    # Run cross-file validation checks.
+    if args.extract or (args.list and args.archive):
+        try:
+            validate.guard_configuration_contains_repository(args.repository, configs)
+        except ValueError as error:
+            yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
+            return
+
+    # Execute the actions corresponding to each configuration file.
+    json_results = []
+    for config_filename, config in configs.items():
+        try:
+            json_results.extend(list(run_configuration(config_filename, config, args)))
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.INFO,
@@ -341,6 +443,9 @@ def collect_configuration_run_summary_logs(config_filenames, args):
             )
             yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
 
+    if json_results:
+        sys.stdout.write(json.dumps(json_results))
+
     if not config_filenames:
         yield logging.makeLogRecord(
             dict(
@@ -350,9 +455,26 @@ def collect_configuration_run_summary_logs(config_filenames, args):
         )
 
 
+def exit_with_help_link():  # pragma: no cover
+    '''
+    Display a link to get help and exit with an error code.
+    '''
+    logger.critical('\nNeed some help? https://torsion.org/borgmatic/#issues')
+    sys.exit(1)
+
+
 def main():  # pragma: no cover
     configure_signals()
-    args = parse_arguments(*sys.argv[1:])
+
+    try:
+        args = parse_arguments(*sys.argv[1:])
+    except ValueError as error:
+        logging.basicConfig(level=logging.CRITICAL, format='%(message)s')
+        logger.critical(error)
+        exit_with_help_link()
+
+    colorama.init(autoreset=True, strip=not should_do_markup(args.no_color))
+
     logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
 
     if args.version:
@@ -369,5 +491,4 @@ def main():  # pragma: no cover
     [logger.handle(log) for log in summary_logs if log.levelno >= logger.getEffectiveLevel()]
 
     if any(log.levelno == logging.CRITICAL for log in summary_logs):
-        logger.critical('\nNeed some help? https://torsion.org/borgmatic/#issues')
-        sys.exit(1)
+        exit_with_help_link()

borgmatic/commands/convert_config.py

@@ -1,13 +1,12 @@
-from argparse import ArgumentParser
 import os
 import sys
 import textwrap
+from argparse import ArgumentParser
 
 from ruamel import yaml
 
 from borgmatic.config import convert, generate, legacy, validate
 
-
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'

borgmatic/commands/generate_config.py

@@ -1,9 +1,8 @@
-from argparse import ArgumentParser
 import sys
+from argparse import ArgumentParser
 
 from borgmatic.config import generate, validate
 
-
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
 
@@ -36,7 +35,7 @@ def main():  # pragma: no cover
 
         print('Generated a sample configuration file at {}.'.format(args.destination_filename))
         print()
-        print('Please edit the file to suit your needs. The values are just representative.')
+        print('Please edit the file to suit your needs. The values are representative.')
         print('All fields are optional except where indicated.')
         print()
         print('If you ever need help: https://torsion.org/borgmatic/#issues')

borgmatic/commands/hook.py

@@ -1,24 +1,34 @@
-import logging
 import subprocess
 
+from borgmatic.logger import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
-def execute_hook(commands, config_filename, description):
+def execute_hook(commands, config_filename, description, dry_run):
+    '''
+
+    Given a list of hook commands to execute, a config filename, a hook description, and whether
+    this is a dry run, run the given commands. Or, don't run them if this is a dry run.
+    '''
     if not commands:
         logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
         return
 
+    dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
+
     if len(commands) == 1:
-        logger.info('{}: Running command for {} hook'.format(config_filename, description))
+        logger.info(
+            '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
+        )
     else:
         logger.info(
-            '{}: Running {} commands for {} hook'.format(
-                config_filename, len(commands), description
+            '{}: Running {} commands for {} hook{}'.format(
+                config_filename, len(commands), description, dry_run_label
             )
         )
 
     for command in commands:
         logger.debug('{}: Hook command: {}'.format(config_filename, command))
-        subprocess.check_call(command, shell=True)
+        if not dry_run:
+            subprocess.check_call(command, shell=True)

borgmatic/commands/validate_config.py

@@ -0,0 +1,57 @@
+import logging
+import sys
+from argparse import ArgumentParser
+
+from borgmatic.config import collect, validate
+from borgmatic.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
+    parser.add_argument(
+        '-c',
+        '--config',
+        nargs='+',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():  # pragma: no cover
+    args = parse_arguments(*sys.argv[1:])
+
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    if len(config_filenames) == 0:
+        logger.critical('No files to validate found')
+        sys.exit(1)
+
+    found_issues = False
+    for config_filename in config_filenames:
+        try:
+            validate.parse_configuration(config_filename, validate.schema_filename())
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logging.critical('{}: Error parsing configuration file'.format(config_filename))
+            logging.critical(error)
+            found_issues = True
+
+    if found_issues:
+        sys.exit(1)
+    else:
+        logger.info(
+            'All given configuration files are valid: {}'.format(', '.join(config_filenames))
+        )

borgmatic/config/collect.py

@@ -20,10 +20,10 @@ def get_default_config_paths():
 
 def collect_config_filenames(config_paths):
     '''
-    Given a sequence of config paths, both filenames and directories, resolve that to just an
-    iterable of files. Accomplish this by listing any given directories looking for contained config
-    files (ending with the ".yaml" extension). This is non-recursive, so any directories within the
-    given directories are ignored.
+    Given a sequence of config paths, both filenames and directories, resolve that to an iterable
+    of files. Accomplish this by listing any given directories looking for contained config files
+    (ending with the ".yaml" extension). This is non-recursive, so any directories within the given
+    directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
     configuration paths. However, skip a default config path if it's missing, so the user doesn't
@@ -41,7 +41,7 @@ def collect_config_filenames(config_paths):
             yield path
             continue
 
-        for filename in os.listdir(path):
+        for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
             if full_filename.endswith('.yaml') and not os.path.isdir(full_filename):
                 yield full_filename

borgmatic/config/convert.py

@@ -64,9 +64,9 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
     return destination_config
 
 
-class LegacyConfigurationNotUpgraded(FileNotFoundError):
+class Legacy_configuration_not_upgraded(FileNotFoundError):
     def __init__(self):
-        super(LegacyConfigurationNotUpgraded, self).__init__(
+        super(Legacy_configuration_not_upgraded, self).__init__(
             '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
 to YAML. This better supports validation, and has a more natural way to express
 lists of values. To upgrade your existing configuration, run:
@@ -83,7 +83,7 @@ instead of the old one.'''
 def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
     '''
     If legacy source configuration exists but no destination upgraded configs do, raise
-    LegacyConfigurationNotUpgraded.
+    Legacy_configuration_not_upgraded.
 
     The idea is that we want to alert the user about upgrading their config if they haven't already.
     '''
@@ -92,22 +92,4 @@ def guard_configuration_upgraded(source_config_filename, destination_config_file
     )
 
     if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise LegacyConfigurationNotUpgraded()
-
-
-class LegacyExcludesFilenamePresent(FileNotFoundError):
-    def __init__(self):
-        super(LegacyExcludesFilenamePresent, self).__init__(
-            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
-to YAML. This better supports validation, and has a more natural way to express
-lists of values. The new configuration file incorporates excludes, so you no
-longer need to provide an excludes filename on the command-line with an
-"--excludes" argument.
-
-Please remove the "--excludes" argument and run borgmatic again.'''
-        )
-
-
-def guard_excludes_filename_omitted(excludes_filename):
-    if excludes_filename is not None:
-        raise LegacyExcludesFilenamePresent()
+        raise Legacy_configuration_not_upgraded()

borgmatic/config/generate.py

@@ -2,7 +2,6 @@ import os
 
 from ruamel import yaml
 
-
 INDENT = 4
 
 

borgmatic/config/legacy.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict, namedtuple
 from configparser import RawConfigParser
 
-
 Section_format = namedtuple('Section_format', ('name', 'options'))
 Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
 

borgmatic/config/load.py

@@ -0,0 +1,62 @@
+import os
+
+import ruamel.yaml
+
+from borgmatic.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def load_configuration(filename):
+    '''
+    Load the given configuration file and return its contents as a data structure of nested dicts
+    and lists.
+
+    Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
+    if there are too many recursive includes.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.Constructor = Include_constructor
+
+    return yaml.load(open(filename))
+
+
+def include_configuration(loader, filename_node):
+    '''
+    Load the given YAML filename (ignoring the given loader so we can use our own), and return its
+    contents as a data structure of nested dicts and lists.
+    '''
+    return load_configuration(os.path.expanduser(filename_node.value))
+
+
+class Include_constructor(ruamel.yaml.SafeConstructor):
+    '''
+    A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
+    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    '''
+
+    def __init__(self, preserve_quotes=None, loader=None):
+        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        self.add_constructor('!include', include_configuration)
+
+    def flatten_mapping(self, node):
+        '''
+        Support the special case of shallow merging included configuration into an existing mapping
+        using the YAML '<<' merge key. Example syntax:
+
+        ```
+        retention:
+            keep_daily: 1
+            <<: !include common.yaml
+        ```
+        '''
+        representer = ruamel.yaml.representer.SafeRepresenter()
+
+        for index, (key_node, value_node) in enumerate(node.value):
+            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+                included_value = representer.represent_mapping(
+                    tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
+                )
+                node.value[index] = (key_node, included_value)
+
+        super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -32,6 +32,10 @@ map:
                 type: bool
                 desc: Stay in same file system (do not cross mount points). Defaults to false.
                 example: true
+            numeric_owner:
+                type: bool
+                desc: Only store/extract numeric user and group identifiers. Defaults to false.
+                example: true
             read_special:
                 type: bool
                 desc: |
@@ -161,9 +165,35 @@ map:
             ssh_command:
                 type: scalar
                 desc: |
-                    Command to use instead of just "ssh". This can be used to specify ssh options.
+                    Command to use instead of "ssh". This can be used to specify ssh options.
                     Defaults to not set.
                 example: ssh -i /path/to/private/key
+            borg_base_directory:
+                type: scalar
+                desc: |
+                    Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
+                    See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
+                example: /path/to/base
+            borg_config_directory:
+                type: scalar
+                desc: |
+                    Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
+                example: /path/to/base/config
+            borg_cache_directory:
+                type: scalar
+                desc: |
+                    Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
+                example: /path/to/base/cache
+            borg_security_directory:
+                type: scalar
+                desc: |
+                    Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
+                example: /path/to/base/config/security
+            borg_keys_directory:
+                type: scalar
+                desc: |
+                    Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
+                example: /path/to/base/config/keys
             umask:
                 type: scalar
                 desc: Umask to be used for borg create. Defaults to 0077.
@@ -243,7 +273,7 @@ map:
                     "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
                     all consistency checks. "repository" checks the consistency of the repository,
                     "archive" checks all of the archives, and "extract" does an extraction dry-run
-                    of just the most recent archive.
+                    of the most recent archive.
                 example:
                     - repository
                     - archives
@@ -281,16 +311,16 @@ map:
                     - type: scalar
                 desc: List of one or more shell commands or scripts to execute before creating a backup.
                 example:
-                    - echo "`date` - Starting a backup job."
+                    - echo "Starting a backup job."
             after_backup:
                 seq:
                     - type: scalar
                 desc: List of one or more shell commands or scripts to execute after creating a backup.
                 example:
-                    - echo "`date` - Backup created."
+                    - echo "Backup created."
             on_error:
                 seq:
                     - type: scalar
                 desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
                 example:
-                    - echo "`date` - Error while creating a backup."
+                    - echo "Error while creating a backup."

borgmatic/config/validate.py

@@ -3,10 +3,12 @@ import logging
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-from ruamel import yaml
+import ruamel.yaml
 
+from borgmatic.config import load
+from borgmatic.logger import get_logger
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 def schema_filename():
@@ -87,13 +89,13 @@ def parse_configuration(config_filename, schema_filename):
     logging.getLogger('pykwalify').setLevel(logging.ERROR)
 
     try:
-        config = yaml.safe_load(open(config_filename))
-        schema = yaml.safe_load(open(schema_filename))
-    except yaml.error.YAMLError as error:
+        config = load.load_configuration(config_filename)
+        schema = load.load_configuration(schema_filename)
+    except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
     # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
-    # simply remove all examples before passing the schema to pykwalify.
+    # remove all examples before passing the schema to pykwalify.
     for section_name, section_schema in schema['map'].items():
         for field_name, field_schema in section_schema['map'].items():
             field_schema.pop('example', None)
@@ -107,3 +109,46 @@ def parse_configuration(config_filename, schema_filename):
     apply_logical_validation(config_filename, parsed_result)
 
     return parsed_result
+
+
+def guard_configuration_contains_repository(repository, configurations):
+    '''
+    Given a repository path and a dict mapping from config filename to corresponding parsed config
+    dict, ensure that the repository is declared exactly once in all of the configurations.
+
+    If no repository is given, then error if there are multiple configured repositories.
+
+    Raise ValueError if the repository is not found in a configuration, or is declared multiple
+    times.
+    '''
+    if not repository:
+        count = len(
+            tuple(
+                config_repository
+                for config in configurations.values()
+                for config_repository in config['location']['repositories']
+            )
+        )
+
+        if count > 1:
+            raise ValueError(
+                'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
+                    repository
+                )
+            )
+
+        return
+
+    count = len(
+        tuple(
+            config_repository
+            for config in configurations.values()
+            for config_repository in config['location']['repositories']
+            if repository == config_repository
+        )
+    )
+
+    if count == 0:
+        raise ValueError('Repository {} not found in configuration files'.format(repository))
+    if count > 1:
+        raise ValueError('Repository {} found in multiple configuration files'.format(repository))

borgmatic/logger.py

@@ -0,0 +1,99 @@
+import logging
+import os
+import sys
+
+import colorama
+
+
+def to_bool(arg):
+    '''
+    Return a boolean value based on `arg`.
+    '''
+    if arg is None or isinstance(arg, bool):
+        return arg
+
+    if isinstance(arg, str):
+        arg = arg.lower()
+
+    if arg in ('yes', 'on', '1', 'true', 1):
+        return True
+
+    return False
+
+
+def should_do_markup(no_color):
+    '''
+    Determine if we should enable colorama marking up.
+    '''
+    if no_color:
+        return False
+
+    py_colors = os.environ.get('PY_COLORS', None)
+
+    if py_colors is not None:
+        return to_bool(py_colors)
+
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+
+
+LOG_LEVEL_TO_COLOR = {
+    logging.CRITICAL: colorama.Fore.RED,
+    logging.ERROR: colorama.Fore.RED,
+    logging.WARN: colorama.Fore.YELLOW,
+    logging.INFO: colorama.Fore.GREEN,
+    logging.DEBUG: colorama.Fore.CYAN,
+}
+
+
+class Borgmatic_logger(logging.Logger):
+    def critical(self, msg, *args, **kwargs):
+        color = LOG_LEVEL_TO_COLOR.get(logging.CRITICAL)
+
+        return super(Borgmatic_logger, self).critical(color_text(color, msg), *args, **kwargs)
+
+    def error(self, msg, *args, **kwargs):
+        color = LOG_LEVEL_TO_COLOR.get(logging.ERROR)
+
+        return super(Borgmatic_logger, self).error(color_text(color, msg), *args, **kwargs)
+
+    def warn(self, msg, *args, **kwargs):
+        color = LOG_LEVEL_TO_COLOR.get(logging.WARN)
+
+        return super(Borgmatic_logger, self).warn(color_text(color, msg), *args, **kwargs)
+
+    def info(self, msg, *args, **kwargs):
+        color = LOG_LEVEL_TO_COLOR.get(logging.INFO)
+
+        return super(Borgmatic_logger, self).info(color_text(color, msg), *args, **kwargs)
+
+    def debug(self, msg, *args, **kwargs):
+        color = LOG_LEVEL_TO_COLOR.get(logging.DEBUG)
+
+        return super(Borgmatic_logger, self).debug(color_text(color, msg), *args, **kwargs)
+
+    def handle(self, record):
+        color = LOG_LEVEL_TO_COLOR.get(record.levelno)
+        colored_record = logging.makeLogRecord(
+            dict(levelno=record.levelno, msg=color_text(color, record.msg))
+        )
+
+        return super(Borgmatic_logger, self).handle(colored_record)
+
+
+def get_logger(name=None):
+    '''
+    Build a logger with the given name.
+    '''
+    logging.setLoggerClass(Borgmatic_logger)
+    logger = logging.getLogger(name)
+    return logger
+
+
+def color_text(color, message):
+    '''
+    Give colored text.
+    '''
+    if not color:
+        return message
+
+    return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)

borgmatic/verbosity.py

@@ -1,6 +1,5 @@
 import logging
 
-
 VERBOSITY_WARNING = 0
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2

docs/how-to/deal-with-very-large-backups.md

@@ -0,0 +1,96 @@
+---
+title: How to deal with very large backups
+---
+## Biggish data
+
+Borg itself is great for efficiently de-duplicating data across successive
+backup archives, even when dealing with very large repositories. But you may
+find that while borgmatic's default mode of "prune, create, and check" works
+well on small repositories, it's not so great on larger ones. That's because
+running the default consistency checks takes a long time on large
+repositories.
+
+### A la carte actions
+
+If you find yourself in this situation, you have some options. First, you can
+run borgmatic's pruning, creating, or checking actions separately. For
+instance, the the following optional flags are available:
+
+```bash
+borgmatic --prune
+borgmatic --create
+borgmatic --check
+```
+
+You can run with only one of these flags provided, or you can mix and match
+any number of them in a single borgmatic run. This supports approaches like
+making backups with `--create` on a frequent schedule, while only running
+expensive consistency checks with `--check` on a much less frequent basis from
+a separate cron job.
+
+### Consistency check configuration
+
+Another option is to customize your consistency checks. The default
+consistency checks run both full-repository checks and per-archive checks
+within each repository.
+
+But if you find that archive checks are too slow, for example, you can
+configure borgmatic to run repository checks only. Configure this in the
+`consistency` section of borgmatic configuration:
+
+```yaml
+consistency:
+    checks:
+        - repository
+```
+
+If that's still too slow, you can disable consistency checks entirely,
+either for a single repository or for all repositories.
+
+Disabling all consistency checks looks like this:
+
+```yaml
+consistency:
+    checks:
+        - disabled
+```
+
+Or, if you have multiple repositories in your borgmatic configuration file,
+you can keep running consistency checks, but only against a subset of the
+repositories:
+
+```yaml
+consistency:
+    check_repositories:
+        - path/of/repository_to_check.borg
+```
+
+
+## Troubleshooting
+
+### Broken pipe with remote repository
+
+When running borgmatic on a large remote repository, you may receive errors
+like the following, particularly while "borg check" is validating backups for
+consistency:
+
+```text
+    Write failed: Broken pipe
+    borg: Error: Connection closed by remote host
+```
+
+This error can be caused by an ssh timeout, which you can rectify by adding
+the following to the `~/.ssh/config` file on the client:
+
+```text
+    Host *
+        ServerAliveInterval 120
+```
+
+This should make the client keep the connection alive while validating
+backups.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)

docs/how-to/develop-on-borgmatic.md

@@ -0,0 +1,103 @@
+---
+title: How to develop on borgmatic
+---
+## Source code
+
+To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
+
+```bash
+git clone https://projects.torsion.org/witten/borgmatic.git
+```
+
+Or:
+
+```bash
+git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git
+```
+
+Then, install borgmatic
+"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)"
+so that you can run borgmatic commands while you're hacking on them to
+make sure your changes work.
+
+```bash
+cd borgmatic/
+pip3 install --editable --user .
+```
+
+Note that this will typically install the borgmatic commands into
+`~/.local/bin`, which may or may not be on your PATH. There are other ways to
+install borgmatic editable as well, for instance into the system Python
+install (so without `--user`, as root), or even into a
+[virtualenv](https://virtualenv.pypa.io/en/stable/). How or where you install
+borgmatic is up to you, but generally an editable install makes development
+and testing easier.
+
+
+## Automated tests
+
+Assuming you've cloned the borgmatic source code as described above, and
+you're in the `borgmatic/` working copy, install tox, which is used for
+setting up testing environments:
+
+```bash
+pip3 install --user tox
+```
+
+Finally, to actually run tests, run:
+
+```bash
+cd borgmatic
+tox
+```
+
+### Code formatting
+
+If when running tests, you get an error from the
+[Black](https://black.readthedocs.io/en/stable/) code formatter about files
+that would be reformatted, you can ask Black to format them for you via the
+following:
+
+```bash
+tox -e black
+```
+
+### End-to-end tests
+
+borgmatic additionally includes some end-to-end tests that integration test
+with Borg for a few representative scenarios. These tests don't run by default
+because they're relatively slow and depend on Borg. If you would like to run
+them:
+
+```bash
+tox -e end-to-end
+```
+
+## Code style
+
+Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
+the following deviations from it:
+
+ * For strings, prefer single quotes over double quotes.
+ * Limit all lines to a maximum of 100 characters.
+ * Use trailing commas within multiline values or argument lists.
+ * For multiline constructs, put opening and closing delimeters on lines
+   separate from their contents.
+ * Within multiline constructs, use standard four-space indentation. Don't align
+   indentation with an opening delimeter.
+
+borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
+formatter and [Flake8](http://flake8.pycqa.org/en/latest/) code checker, so
+certain code style requirements will be enforced when running automated tests.
+See the Black and Flake8 documentation for more information.
+
+## Continuous integration
+
+Each pull request triggers a continuous integration build which runs the test
+suite. You can view these builds on
+[build.torsion.org](https://build.torsion.org/witten/borgmatic), and they're
+also linked from the commits list on each pull request.
+
+## Related documentation
+
+ * [Inspect your backups](../../docs/how-to/inspect-your-backups.md)

docs/how-to/inspect-your-backups.md

@@ -0,0 +1,46 @@
+---
+title: How to inspect your backups
+---
+## Backup progress
+
+By default, borgmatic runs proceed silently except in the case of errors. But
+if you'd like to to get additional information about the progress of the
+backup as it proceeds, use the verbosity option:
+
+```bash
+borgmatic --verbosity 1
+```
+
+This lists the files that borgmatic is archiving, which are those that are new
+or changed since the last backup.
+
+Or, for even more progress and debug spew:
+
+```bash
+borgmatic --verbosity 2
+```
+
+## Existing backups
+
+Borgmatic provides convenient flags for Borg's
+[list](https://borgbackup.readthedocs.io/en/stable/usage/list.html) and
+[info](https://borgbackup.readthedocs.io/en/stable/usage/info.html)
+functionality:
+
+
+```bash
+borgmatic --list
+borgmatic --info
+```
+
+## Scripting borgmatic
+
+To consume the output of borgmatic in other software, you can include an
+optional `--json` flag with `--create`, `--list`, or `--info` to get the
+output formatted as JSON.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
+ * [Develop on borgmatic](../../docs/how-to/develop-on-borgmatic.md)

docs/how-to/make-per-application-backups.md

@@ -0,0 +1,111 @@
+---
+title: How to make per-application backups
+---
+## Multiple backup configurations
+
+You may find yourself wanting to create different backup policies for
+different applications on your system. For instance, you may want one backup
+configuration for your database data directory, and a different configuration
+for your user home directories.
+
+The way to accomplish that is pretty simple: Create multiple separate
+configuration files and place each one in a `/etc/borgmatic.d/` directory. For
+instance:
+
+```bash
+sudo mkdir /etc/borgmatic.d
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+```
+
+When you set up multiple configuration files like this, borgmatic will run
+each one in turn from a single borgmatic invocation. This includes, by
+default, the traditional `/etc/borgmatic/config.yaml` as well.
+
+And if you need even more customizability, you can specify alternate
+configuration paths on the command-line with borgmatic's `--config` option.
+See `borgmatic --help` for more information.
+
+
+## Configuration includes
+
+Once you have multiple different configuration files, you might want to share
+common configuration options across these files with having to copy and paste
+them. To achieve this, you can put fragments of common configuration options
+into a file, and then include or inline that file into one or more borgmatic
+configuration files.
+
+Let's say that you want to include common retention configuration across all
+of your configuration files. You could do that in each configuration file with
+the following:
+
+```yaml
+location:
+   ...
+
+retention:
+    !include /etc/borgmatic/common_retention.yaml
+```
+
+And then the contents of `common_retention.yaml` could be:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+To prevent borgmatic from trying to load these configuration fragments by
+themselves and complaining that they are not valid configuration files, you
+should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
+is fine.)
+
+Note that this form of include must be a YAML value rather than a key. For
+example, this will not work:
+
+```yaml
+location:
+   ...
+
+# Don't do this. It won't work!
+!include /etc/borgmatic/common_retention.yaml
+```
+
+But if you do want to merge in a YAML key and its values, keep reading!
+
+
+## Include merging
+
+If you need to get even fancier and pull in common configuration options while
+potentially overriding individual options, you can perform a YAML merge of
+included configuration using the YAML `<<` key. For instance, here's an
+example of a main configuration file that pulls in two retention options via
+an include, and then overrides one of them locally:
+
+```yaml
+location:
+   ...
+
+retention:
+    keep_daily: 5
+    <<: !include /etc/borgmatic/common_retention.yaml
+```
+
+This is what `common_retention.yaml` might look like:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration would have a
+`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
+
+When there is a collision of an option between the local file and the merged
+include, the local file's option takes precedent. And note that this is a
+shallow merge rather than a deep merge, so the merging does not descend into
+nested values.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)

docs/how-to/restore-a-backup.md

@@ -0,0 +1,61 @@
+---
+title: How to restore a backup
+---
+## Extract
+
+When the worst happens—or you want to test your backups—the first step is
+to figure out which archive to restore. A good way to do that is to use the
+`--list` action:
+
+```bash
+borgmatic --list
+```
+
+That should yield output looking something like:
+
+```text
+host-2019-01-01T04:05:06.070809      Tue, 2019-01-01 04:05:06 [...]
+host-2019-01-02T04:06:07.080910      Wed, 2019-01-02 04:06:07 [...]
+```
+
+Assuming that you want to restore the archive with the most up-to-date files
+and therefore the latest timestamp, run a command like:
+
+```bash
+borgmatic --extract --archive host-2019-01-02T04:06:07.080910
+```
+
+The `--archive` value is the name of the archive to restore. This extracts the
+entire contents of the archive to the current directory, so make sure you're
+in the right place before running the command.
+
+
+## Repository selection
+
+If you have a single repository in your borgmatic configuration file(s), no
+problem: the `--extract` action figures out which repository to use.
+
+But if you have multiple repositories configured, then you'll need to specify
+the repository path containing the archive to extract. Here's an example:
+
+```bash
+borgmatic --extract --repository repo.borg --archive host-2019-...
+```
+
+## Restore particular files
+
+Sometimes, you want to restore a single deleted file, rather than restoring
+everything from an archive. To do that, tack on one or more `--restore-path`
+values. For instance:
+
+```bash
+borgmatic --extract --archive host-2019-... --restore-path /path/1 /path/2
+```
+
+Like a whole-archive restore, this also restores into the current directory.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
+ * [Inspect your backups](../../docs/how-to/inspect-your-backups.md)

docs/how-to/run-preparation-steps-before-backups.md

@@ -0,0 +1,51 @@
+---
+title: How to run preparation steps before backups
+---
+## Preparation and cleanup hooks
+
+If you find yourself performing prepraration tasks before your backup runs, or
+cleanup work afterwards, borgmatic hooks may be of interest. Hooks are
+shell commands that borgmatic executes for you at various points, and they're
+configured in the `hooks` section of your configuration file.
+
+For instance, you can specify `before_backup` hooks to dump a database to file
+before backing it up, and specify `after_backup` hooks to delete the temporary
+file afterwards. Here's an example:
+
+```yaml
+hooks:
+    before_backup:
+        - dump-a-database /to/file.sql
+    after_backup:
+        - rm /to/file.sql
+```
+
+borgmatic hooks run once per configuration file. `before_backup` hooks run
+prior to backups of all repositories. `after_backup` hooks run afterwards, but
+not if an error occurs in a previous hook or in the backups themselves.
+
+
+## Error hooks
+
+borgmatic also runs `on_error` hooks if an error occurs. Here's an example
+configuration:
+
+```yaml
+hooks:
+    on_error:
+        - echo "Error while creating a backup."
+```
+
+
+## Security
+
+An important security note about hooks: borgmatic executes all hook commands
+with the user permissions of borgmatic itself. So to prevent potential shell
+injection or privilege escalation, do not forget to set secure permissions
+(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
+ * [Make per-application backups](../../docs/how-to/make-per-application-backups.md)

docs/how-to/set-up-backups.md

@@ -0,0 +1,216 @@
+---
+title: How to set up backups with borgmatic
+---
+## Installation
+
+To get up and running, first [install
+Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
+least version 1.1.
+
+Borgmatic consumes configurations in `/etc/borgmatic/` and `/etc/borgmatic.d/`
+by default. Therefore, we show how to install borgmatic for the root user which
+will have access permissions for these locations by default.
+
+Run the following commands to download and install borgmatic:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+This is a [recommended user site
+installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site).
+You will need to ensure that `/root/.local/bin` is available on your `$PATH` so
+that the borgmatic executable is available.
+
+Note that your pip binary may have a different name than "pip3". Make sure
+you're using Python 3, as borgmatic does not support Python 2.
+
+### Other ways to install
+
+Along with the above process, you have several other options for installing
+borgmatic:
+
+ * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
+ * [Docker image with support for scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
+ * [Debian](https://tracker.debian.org/pkg/borgmatic)
+ * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
+ * [Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
+ * [Arch Linux](https://aur.archlinux.org/packages/borgmatic/)
+ * [OpenBSD](http://ports.su/sysutils/borgmatic)
+ * [openSUSE](https://software.opensuse.org/package/borgmatic)
+
+
+## Configuration
+
+After you install borgmatic, generate a sample configuration file:
+
+```bash
+sudo generate-borgmatic-config
+```
+
+If that command is not found, then it may be installed in a location that's
+not in your system `PATH`. Try looking in `/usr/local/bin/`.
+
+This generates a sample configuration file at /etc/borgmatic/config.yaml (by
+default). You should edit the file to suit your needs, as the values are
+representative. All options are optional except where indicated, so feel free
+to ignore anything you don't need.
+
+Note that the configuration file is organized into distinct sections, each
+with a section name like `location:` or `storage:`. So take care that if you
+uncomment a particular option, also uncomment its containing section name, or
+else borgmatic won't recognize the option.
+
+You can also get the same sample configuration file from the [configuration
+reference](../../docs/reference/configuration.md), the authoritative set of
+all configuration options. This is handy if borgmatic has added new options
+since you originally created your configuration file.
+
+
+### Encryption
+
+Note that if you plan to run borgmatic on a schedule with cron, and you
+encrypt your Borg repository with a passphrase instead of a key file, you'll
+either need to set the borgmatic `encryption_passphrase` configuration
+variable or set the `BORG_PASSPHRASE` environment variable. See the
+[repository encryption
+section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
+of the Borg Quick Start for more info.
+
+Alternatively, you can specify the passphrase programatically by setting
+either the borgmatic `encryption_passcommand` configuration variable or the
+`BORG_PASSCOMMAND` environment variable. See the [Borg Security
+FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
+for more info.
+
+
+### Validation
+
+If you'd like to validate that your borgmatic configuration is valid, the
+following command is available for that:
+
+```bash
+sudo validate-borgmatic-config
+```
+
+This command's exit status (`$?` in Bash) is zero when configuration is valid
+and non-zero otherwise.
+
+Validating configuration can be useful if you generate your configuration
+files via configuration management, or you just want to double check that your
+hand edits are valid.
+
+
+## Initialization
+
+Before you can create backups with borgmatic, you first need to initialize a
+Borg repository so you have a destination for your backup archives. (But skip
+this step if you already have a Borg repository.) To create a repository, run
+a command like the following:
+
+```bash
+borgmatic --init --encryption repokey
+```
+
+This uses the borgmatic configuration file you created above to determine
+which local or remote repository to create, and encrypts it with the
+encryption passphrase specified there if one is provided. Read about [Borg
+encryption
+modes](https://borgbackup.readthedocs.io/en/latest/usage/init.html#encryption-modes)
+for the menu of available encryption modes.
+
+Also, optionally check out the [Borg Quick
+Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) for more
+background about repository initialization.
+
+Note that borgmatic skips repository initialization if the repository already
+exists. This supports use cases like ensuring a repository exists prior to
+performing a backup.
+
+If the repository is on a remote host, make sure that your local user has
+key-based SSH access to the desired user account on the remote host.
+
+
+## Backups
+
+Now that you've configured borgmatic and initialized a repository, it's a
+good idea to test that borgmatic is working. So to run borgmatic and start a
+backup, you can invoke it like this:
+
+```bash
+borgmatic --verbosity 1
+```
+
+By default, this will also prune any old backups as per the configured
+retention policy, and check backups for consistency problems due to things
+like file damage.
+
+The verbosity flag makes borgmatic list the files that it's archiving, which
+are those that are new or changed since the last backup. Eyeball the list and
+see if it matches your expectations based on the configuration.
+
+
+## Autopilot
+
+Running backups manually is good for validating your configuration, but I'm
+guessing that you want to run borgmatic automatically, say once a day. To do
+that, you can configure a separate job runner to invoke it periodically.
+
+### cron
+
+If you're using cron, download the [sample cron
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic).
+Then, from the directory where you downloaded it:
+
+```bash
+sudo mv borgmatic /etc/cron.d/borgmatic
+sudo chmod +x /etc/cron.d/borgmatic
+```
+
+You can modify the cron file if you'd like to run borgmatic more or less frequently.
+
+### systemd
+
+If you're using systemd instead of cron to run jobs, download the [sample
+systemd service
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.service)
+and the [sample systemd timer
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.timer).
+Then, from the directory where you downloaded them:
+
+```bash
+sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
+sudo systemctl enable borgmatic.timer
+sudo systemctl start borgmatic.timer
+```
+
+Feel free to modify the timer file based on how frequently you'd like
+borgmatic to run.
+
+## Colored Output
+
+Borgmatic uses [colorama](https://pypi.org/project/colorama/) to produce
+colored terminal output by default. It is disabled when a non-interactive
+terminal is detected (like a cron job). Otherwise, it can be disabled by
+passing `--no-color` or by setting the environment variable `PY_COLORS=False`.
+
+## Troubleshooting
+
+### libyaml compilation errors
+
+borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
+use a C YAML library (libyaml) if present. But if it's not installed, then
+when installing or upgrading borgmatic, you may see errors about compiling the
+YAML library. If so, not to worry. borgmatic should install and function
+correctly even without the C YAML library. And borgmatic won't be any faster
+with the C library present, so you don't need to go out of your way to install
+it.
+
+
+## Related documentation
+
+ * [Make per-application backups](../../docs/how-to/make-per-application-backups.md)
+ * [Deal with very large backups](../../docs/how-to/deal-with-very-large-backups.md)
+ * [Inspect your backups](../../docs/how-to/inspect-your-backups.md)
+ * [borgmatic configuration reference](../../docs/reference/configuration.md)
+ * [borgmatic command-line reference](../../docs/reference/command-line.md)

docs/how-to/upgrade.md

@@ -0,0 +1,79 @@
+---
+title: How to upgrade borgmatic
+---
+## Upgrading
+
+In general, all you should need to do to upgrade borgmatic is run the
+following:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+See below about special cases.
+
+
+### Upgrading from borgmatic 1.0.x
+
+borgmatic changed its configuration file format in version 1.1.0 from
+INI-style to YAML. This better supports validation, and has a more natural way
+to express lists of values. To upgrade your existing configuration, first
+upgrade to the new version of borgmatic.
+
+As of version 1.1.0, borgmatic no longer supports Python 2. If you were
+already running borgmatic with Python 3, then you can upgrade borgmatic
+in-place:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+But if you were running borgmatic with Python 2, uninstall and reinstall instead:
+
+```bash
+sudo pip uninstall borgmatic
+sudo pip3 install --user borgmatic
+```
+
+The pip binary names for different versions of Python can differ, so the above
+commands may need some tweaking to work on your machine.
+
+
+Once borgmatic is upgraded, run:
+
+```bash
+sudo upgrade-borgmatic-config
+```
+
+That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
+(by default) using the values from both your existing configuration and
+excludes files. The new version of borgmatic will consume the YAML
+configuration file instead of the old one.
+
+
+### Upgrading from atticmatic
+
+You can ignore this section if you're not an atticmatic user (the former name
+of borgmatic).
+
+borgmatic only supports Borg now and no longer supports Attic. So if you're
+an Attic user, consider switching to Borg. See the [Borg upgrade
+command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
+for more information. Then, follow the instructions above about setting up
+your borgmatic configuration files.
+
+If you were already using Borg with atticmatic, then you can upgrade
+from atticmatic to borgmatic by running the following commands:
+
+```bash
+sudo pip3 uninstall atticmatic
+sudo pip3 install --user borgmatic
+```
+
+That's it! borgmatic will continue using your /etc/borgmatic configuration
+files.
+
+
+## Related documentation
+
+ * [Develop on borgmatic](../../docs/how-to/develop-on-borgmatic.md)

docs/reference/command-line.md

@@ -0,0 +1,16 @@
+---
+title: borgmatic command-line reference
+---
+## borgmatic options
+
+Here are all of the available borgmatic command-line options:
+
+```
+{% include borgmatic/command-line.txt %}
+```
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
+ * [borgmatic configuration reference](../../docs/reference/configuration.md)

docs/reference/configuration.md

@@ -0,0 +1,19 @@
+---
+title: borgmatic configuration reference
+---
+## Configuration file
+
+Here is a full sample borgmatic configuration file including all available options: 
+
+```yaml
+{% include borgmatic/config.yaml %}
+```
+
+Note that you can also [download this configuration
+file](../../docs/reference/config.yaml) for use locally.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
+ * [borgmatic command-line reference](../../docs/reference/command-line.md)

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 100
+skip-string-normalization = true

sample/cron-alpine/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * PATH=$PATH:/usr/bin /usr/bin/borgmatic
+0 3 * * * PATH=$PATH:/usr/bin /root/.local/bin/borgmatic

sample/cron/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/borgmatic
+0 3 * * * root PATH=$PATH:/usr/local/bin /root/.local/bin/borgmatic

sample/systemd/borgmatic.service

@@ -3,4 +3,4 @@ Description=borgmatic backup
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/borgmatic
+ExecStart=/root/.local/bin/borgmatic

scripts/black

@@ -1,7 +0,0 @@
-#!/usr/bin/env sh
-
-if which black; then
-    black --skip-string-normalization --line-length 100 --check .
-else
-    echo "Skipping black due to not being installed."
-fi

scripts/pip

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+# Temporary work around for https://github.com/pypa/pip/issues/6434
+python -m pip install --upgrade pip==19.1.1
+python -m pip install --no-use-pep517 $*
+

scripts/push

@@ -2,5 +2,7 @@
 
 set -e
 
-git push -u origin master
-git push -u github master
+branch_name=$(git rev-parse --abbrev-ref HEAD)
+
+git push -u github "$branch_name"
+git push -u origin "$branch_name"

setup.cfg

@@ -1,2 +1,18 @@
 [metadata]
 description-file=README.md
+
+[tool:pytest]
+testpaths = tests
+addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end
+
+[flake8]
+ignore = E501,W503
+exclude = *.*/*
+
+[tool:isort]
+force_single_line = False
+include_trailing_comma = True
+known_first_party = borgmatic
+line_length = 100
+multi_line_output = 3
+skip = .tox

setup.py

@@ -1,7 +1,6 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-
-VERSION = '1.2.14'
+VERSION = '1.3.5'
 
 
 setup(
@@ -20,15 +19,21 @@ setup(
         'Topic :: Security :: Cryptography',
         'Topic :: System :: Archiving :: Backup',
     ],
-    packages=find_packages(),
+    packages=find_packages(exclude=['tests*']),
     entry_points={
         'console_scripts': [
             'borgmatic = borgmatic.commands.borgmatic:main',
             'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
             'generate-borgmatic-config = borgmatic.commands.generate_config:main',
+            'validate-borgmatic-config = borgmatic.commands.validate_config:main',
         ]
     },
     obsoletes=['atticmatic'],
-    install_requires=('pykwalify>=1.6.0,<14.06', 'ruamel.yaml>0.15.0,<0.16.0', 'setuptools'),
+    install_requires=(
+        'pykwalify>=1.6.0,<14.06',
+        'ruamel.yaml>0.15.0,<0.16.0',
+        'setuptools',
+        'colorama>=0.4.1,<0.5',
+    ),
     include_package_data=True,
 )

test_requirements.txt

@@ -3,10 +3,12 @@ atomicwrites==1.2.1
 attrs==18.2.0
 black==18.9b0; python_version >= '3.6'
 Click==7.0
+colorama==0.4.1
 coverage==4.5.1
 docopt==0.6.2
 flake8==3.5.0
 flexmock==0.10.2
+isort==4.3.19
 mccabe==0.6.1
 more-itertools==4.3.0
 pluggy==0.7.1

tests/end-to-end/test_borgmatic.py

@@ -33,6 +33,11 @@ def test_borgmatic_command():
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()
     repository_path = os.path.join(temporary_directory, 'test.borg')
+    extract_path = os.path.join(temporary_directory, 'extract')
+
+    original_working_directory = os.getcwd()
+    os.mkdir(extract_path)
+    os.chdir(extract_path)
 
     try:
         config_path = os.path.join(temporary_directory, 'test.yaml')
@@ -51,8 +56,19 @@ def test_borgmatic_command():
 
         assert len(parsed_output) == 1
         assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
 
-        # Also exercise the info flag.
+        # Extract the created archive into the current (temporary) directory, and confirm that the
+        # extracted file looks right.
+        output = subprocess.check_output(
+            'borgmatic --config {} --extract --archive {}'.format(config_path, archive_name).split(
+                ' '
+            )
+        ).decode(sys.stdout.encoding)
+        extracted_config_path = os.path.join(extract_path, config_path)
+        assert open(extracted_config_path).read() == open(config_path).read()
+
+        # Exercise the info flag.
         output = subprocess.check_output(
             'borgmatic --config {} --info --json'.format(config_path).split(' ')
         ).decode(sys.stdout.encoding)
@@ -61,4 +77,5 @@ def test_borgmatic_command():
         assert len(parsed_output) == 1
         assert 'repository' in parsed_output[0]
     finally:
+        os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)

tests/end-to-end/test_validate_config.py

@@ -0,0 +1,36 @@
+import os
+import subprocess
+import tempfile
+
+
+def test_validate_config_command_with_valid_configuration_succeeds():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 0
+
+
+def test_validate_config_command_with_invalid_configuration_fails():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"')
+        config_file = open(config_path, 'w')
+        config_file.write(config)
+        config_file.close()
+
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 1

tests/integration/commands/test_borgmatic.py

@@ -1,7 +1,7 @@
 import subprocess
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.commands import borgmatic as module
 
@@ -94,6 +94,12 @@ def test_parse_arguments_disallows_encryption_mode_without_init():
         module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
 
 
+def test_parse_arguments_allows_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey')
+
+
 def test_parse_arguments_requires_encryption_mode_with_init():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
@@ -136,44 +142,128 @@ def test_parse_arguments_disallows_init_and_dry_run():
         )
 
 
+def test_parse_arguments_disallows_repository_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
+
+
+def test_parse_arguments_allows_repository_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments(
+        '--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
+    )
+
+
+def test_parse_arguments_allows_repository_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
+
+
+def test_parse_arguments_disallows_archive_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--archive', 'test')
+
+
+def test_parse_arguments_disallows_restore_paths_without_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--restore-path', 'test')
+
+
+def test_parse_arguments_allows_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
+
+
+def test_parse_arguments_requires_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--extract')
+
+
 def test_parse_arguments_allows_progress_and_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--progress', '--create', '--list')
 
 
+def test_parse_arguments_allows_progress_and_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
+
+
 def test_parse_arguments_disallows_progress_without_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--progress', '--list')
 
 
 def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--stats', '--create', '--list')
 
 
 def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--stats', '--prune', '--list')
 
 
 def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--stats', '--list')
 
 
+def test_parse_arguments_with_just_stats_flag_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats')
+
+
 def test_parse_arguments_allows_json_with_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--list', '--json')
     module.parse_arguments('--info', '--json')
 
 
 def test_parse_arguments_disallows_json_without_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--json')
 
 
 def test_parse_arguments_disallows_json_with_both_list_and_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--list', '--info', '--json')
 
 
 def test_borgmatic_version_matches_news_version():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
     news_version = open('NEWS').readline()
 

tests/integration/commands/test_convert_config.py

@@ -1,7 +1,7 @@
 import os
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.commands import convert_config as module
 

tests/integration/commands/test_validate_config.py

@@ -0,0 +1,20 @@
+from flexmock import flexmock
+
+from borgmatic.commands import validate_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_paths == config_paths
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    assert parser.config_paths == ['myconfig', 'otherconfig']

tests/integration/config/test_generate.py

@@ -1,9 +1,9 @@
-from io import StringIO
 import os
 import sys
+from io import StringIO
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import generate as module
 

tests/integration/config/test_legacy.py

@@ -1,7 +1,6 @@
-from io import StringIO
-
-from collections import OrderedDict
 import string
+from collections import OrderedDict
+from io import StringIO
 
 from borgmatic.config import legacy as module
 

tests/integration/config/test_load.py

@@ -0,0 +1,40 @@
+import sys
+
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_load_configuration_parses_contents():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_inlines_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return('value')
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        'key: !include include.yaml'
+    )
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_merges_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        foo: bar
+        baz: quux
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: override
+        <<: !include include.yaml
+        '''
+    )
+
+    assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}

tests/integration/config/test_validate.py

@@ -2,8 +2,8 @@ import io
 import string
 import sys
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import validate as module
 
@@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
     module.parse_configuration('config.yaml', 'schema.yaml')
 
 
+def test_parse_configuration_inlines_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24},
+    }
+
+
+def test_parse_configuration_merges_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_daily: 1
+            <<: !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 1, 'keep_hourly': 24},
+    }
+
+
 def test_parse_configuration_raises_for_missing_config_file():
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')

tests/unit/borg/test_check.py

@@ -1,11 +1,12 @@
-from subprocess import STDOUT
 import logging
 import sys
+from subprocess import STDOUT
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.borg import check as module
+
 from ..test_verbosity import insert_logging_mock
 
 

tests/unit/borg/test_create.py

@@ -3,6 +3,7 @@ import logging
 from flexmock import flexmock
 
 from borgmatic.borg import create as module
+
 from ..test_verbosity import insert_logging_mock
 
 
@@ -43,6 +44,21 @@ def test_expand_directories_considers_none_as_no_directories():
     assert paths == ()
 
 
+def test_expand_home_directories_expands_tildes():
+    flexmock(module.os.path).should_receive('expanduser').with_args('~/bar').and_return('/foo/bar')
+    flexmock(module.os.path).should_receive('expanduser').with_args('baz').and_return('baz')
+
+    paths = module._expand_home_directories(('~/bar', 'baz'))
+
+    assert paths == ('/foo/bar', 'baz')
+
+
+def test_expand_home_directories_considers_none_as_no_directories():
+    paths = module._expand_home_directories(None)
+
+    assert paths == ()
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
     flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@@ -54,11 +70,6 @@ def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
     module._write_pattern_file([])
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
 def test_make_pattern_flags_includes_pattern_filename_when_given():
     pattern_flags = module._make_pattern_flags(
         location_config={'patterns': ['R /', '- /var']}, pattern_filename='/tmp/patterns'
@@ -149,11 +160,14 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo
 
 
 def test_create_archive_calls_borg_with_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -169,13 +183,16 @@ def test_create_archive_calls_borg_with_parameters():
 
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
         flexmock(name='/tmp/patterns')
     ).and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + pattern_flags)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + pattern_flags, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -191,15 +208,16 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(
-        ('exclude',)
-    )
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
         flexmock(name='/tmp/excludes')
     )
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
-    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + exclude_flags, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -214,12 +232,15 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--stats'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--stats'), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
 
     module.create_archive(
@@ -235,12 +256,14 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
 
 
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc')
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc'),
+        capture_output=False,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -257,12 +280,15 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
 
 
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--dry-run',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--dry-run',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=True,
@@ -277,14 +303,17 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
 
 
 def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter():
-    """ --dry-run and --stats are mutually exclusive, see:
-    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    # --dry-run and --stats are mutually exclusive, see:
+    # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--dry-run'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--dry-run'), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
 
     module.create_archive(
@@ -300,15 +329,17 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
 
 
 def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter():
-    """ --dry-run and --stats are mutually exclusive, see:
-    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    # --dry-run and --stats are mutually exclusive, see:
+    # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run')
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run'),
+        capture_output=False,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -325,11 +356,14 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
 
 
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--checkpoint-interval', '600'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--checkpoint-interval', '600'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -344,11 +378,14 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
 
 
 def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--chunker-params', '1,2,3,4'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--chunker-params', '1,2,3,4'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -363,11 +400,14 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
 
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--compression', 'rle'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -382,11 +422,14 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--remote-ratelimit', '100'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -400,12 +443,15 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     )
 
 
-def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--one-file-system',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -420,12 +466,38 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     )
 
 
-def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--read-special',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--numeric-owner',), capture_output=False
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'numeric_owner': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--read-special',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -441,11 +513,14 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
 
 
 def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -461,11 +536,14 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame
 
 
 def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--nobsdflags',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--nobsdflags',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -481,11 +559,14 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete
 
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--files-cache', 'ctime,size'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -501,11 +582,14 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg1',) + CREATE_COMMAND[1:])
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1',) + CREATE_COMMAND[1:], capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -521,11 +605,14 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -541,11 +628,14 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--umask', '740'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -560,11 +650,14 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--lock-wait', '5'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -579,13 +672,16 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
 
 
 def test_create_archive_with_json_calls_borg_with_json_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--json',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--json',), capture_output=True
+    ).and_return('[]')
 
-    module.create_archive(
+    json_output = module.create_archive(
         dry_run=False,
         repository='repo',
         location_config={
@@ -597,16 +693,18 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
         json=True,
     )
 
+    assert json_output == '[]'
+
 
 def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(
-        ()
-    )
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
+        capture_output=False,
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -623,11 +721,14 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'), capture_output=False
+    )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
@@ -643,14 +744,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(
-        ()
-    )
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
+        capture_output=False,
     )
 
     module.create_archive(
@@ -666,11 +767,14 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -685,11 +789,14 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,

tests/unit/borg/test_environment.py

@@ -42,8 +42,7 @@ def test_initialize_without_configuration_should_not_set_environment():
     try:
         os.environ = {}
         module.initialize({})
-        assert os.environ.get('BORG_PASSCOMMAND') is None
-        assert os.environ.get('BORG_PASSPHRASE') is None
-        assert os.environ.get('BORG_RSH') is None
+
+        assert sum(1 for key in os.environ.keys() if key.startswith('BORG_')) == 0
     finally:
         os.environ = orig_environ

tests/unit/borg/test_execute.py

@@ -0,0 +1,26 @@
+from flexmock import flexmock
+
+from borgmatic.borg import execute as module
+
+
+def test_execute_command_calls_full_command():
+    full_command = ['foo', 'bar']
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(full_command).once()
+
+    output = module.execute_command(full_command)
+
+    assert output is None
+
+
+def test_execute_command_captures_output():
+    full_command = ['foo', 'bar']
+    expected_output = '[]'
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_output').with_args(full_command).and_return(
+        flexmock(decode=lambda: expected_output)
+    ).once()
+
+    output = module.execute_command(full_command, capture_output=True)
+
+    assert output == expected_output

tests/unit/borg/test_extract.py

@@ -4,6 +4,7 @@ import sys
 from flexmock import flexmock
 
 from borgmatic.borg import extract as module
+
 from ..test_verbosity import insert_logging_mock
 
 
@@ -24,7 +25,7 @@ def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
     ).once()
 
 
-def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
+def test_extract_last_archive_dry_run_calls_borg_with_last_archive():
     flexmock(sys.stdout).encoding = 'utf-8'
     insert_subprocess_check_output_mock(
         ('borg', 'list', '--short', 'repo'), result='archive1\narchive2\n'.encode('utf-8')
@@ -44,7 +45,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_bail():
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
 
 
-def test_extract_last_archive_dry_run_with_log_info_should_call_borg_with_info_parameter():
+def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_parameter():
     flexmock(sys.stdout).encoding = 'utf-8'
     insert_subprocess_check_output_mock(
         ('borg', 'list', '--short', 'repo', '--info'), result='archive1\narchive2\n'.encode('utf-8')
@@ -55,7 +56,7 @@ def test_extract_last_archive_dry_run_with_log_info_should_call_borg_with_info_p
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
 
 
-def test_extract_last_archive_dry_run_with_log_debug_should_call_borg_with_debug_parameter():
+def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(sys.stdout).encoding = 'utf-8'
     insert_subprocess_check_output_mock(
         ('borg', 'list', '--short', 'repo', '--debug', '--show-rc'),
@@ -69,7 +70,7 @@ def test_extract_last_archive_dry_run_with_log_debug_should_call_borg_with_debug
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None)
 
 
-def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
+def test_extract_last_archive_dry_run_calls_borg_via_local_path():
     flexmock(sys.stdout).encoding = 'utf-8'
     insert_subprocess_check_output_mock(
         ('borg1', 'list', '--short', 'repo'), result='archive1\narchive2\n'.encode('utf-8')
@@ -79,7 +80,7 @@ def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None, local_path='borg1')
 
 
-def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
+def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters():
     flexmock(sys.stdout).encoding = 'utf-8'
     insert_subprocess_check_output_mock(
         ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
@@ -92,7 +93,7 @@ def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_paramete
     module.extract_last_archive_dry_run(repository='repo', lock_wait=None, remote_path='borg1')
 
 
-def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters():
+def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters():
     flexmock(sys.stdout).encoding = 'utf-8'
     insert_subprocess_check_output_mock(
         ('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
@@ -101,3 +102,124 @@ def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters
     insert_subprocess_mock(('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'))
 
     module.extract_last_archive_dry_run(repository='repo', lock_wait=5)
+
+
+def test_extract_archive_calls_borg_with_restore_path_parameters():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', 'path1', 'path2'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=['path1', 'path2'],
+        location_config={},
+        storage_config={},
+    )
+
+
+def test_extract_archive_calls_borg_with_remote_path_parameters():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--remote-path', 'borg1'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
+        remote_path='borg1',
+    )
+
+
+def test_extract_archive_calls_borg_with_numeric_owner_parameter():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--numeric-owner'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={'numeric_owner': True},
+        storage_config={},
+    )
+
+
+def test_extract_archive_calls_borg_with_umask_parameters():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--umask', '0770'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={'umask': '0770'},
+    )
+
+
+def test_extract_archive_calls_borg_with_lock_wait_parameters():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--lock-wait', '5'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={'lock_wait': '5'},
+    )
+
+
+def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--info'))
+    insert_logging_mock(logging.INFO)
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
+    )
+
+
+def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--debug', '--list', '--show-rc'))
+    insert_logging_mock(logging.DEBUG)
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
+    )
+
+
+def test_extract_archive_calls_borg_with_dry_run_parameter():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--dry-run'))
+
+    module.extract_archive(
+        dry_run=True,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
+    )
+
+
+def test_extract_archive_calls_borg_with_progress_parameter():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--progress'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
+        progress=True,
+    )

tests/unit/borg/test_info.py

@@ -3,56 +3,65 @@ import logging
 from flexmock import flexmock
 
 from borgmatic.borg import info as module
+
 from ..test_verbosity import insert_logging_mock
 
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(check_call_command, **kwargs).once()
-
-
 INFO_COMMAND = ('borg', 'info', 'repo')
 
 
 def test_display_archives_info_calls_borg_with_parameters():
-    insert_subprocess_mock(INFO_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(INFO_COMMAND, capture_output=False)
 
     module.display_archives_info(repository='repo', storage_config={})
 
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
-    insert_subprocess_mock(INFO_COMMAND + ('--info',))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--info',), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
     module.display_archives_info(repository='repo', storage_config={})
 
 
 def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
-    insert_subprocess_mock(INFO_COMMAND + ('--debug', '--show-rc'))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--debug', '--show-rc'), capture_output=False
+    )
     insert_logging_mock(logging.DEBUG)
 
     module.display_archives_info(repository='repo', storage_config={})
 
 
 def test_display_archives_info_with_json_calls_borg_with_json_parameter():
-    insert_subprocess_mock(INFO_COMMAND + ('--json',))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--json',), capture_output=True
+    ).and_return('[]')
 
-    module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+
+    assert json_output == '[]'
 
 
 def test_display_archives_info_with_local_path_calls_borg_via_local_path():
-    insert_subprocess_mock(('borg1',) + INFO_COMMAND[1:])
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1',) + INFO_COMMAND[1:], capture_output=False
+    )
 
     module.display_archives_info(repository='repo', storage_config={}, local_path='borg1')
 
 
 def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_parameters():
-    insert_subprocess_mock(INFO_COMMAND + ('--remote-path', 'borg1'))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+    )
 
     module.display_archives_info(repository='repo', storage_config={}, remote_path='borg1')
 
 
 def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
-    insert_subprocess_mock(INFO_COMMAND + ('--lock-wait', '5'))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--lock-wait', '5'), capture_output=False
+    )
 
     module.display_archives_info(repository='repo', storage_config=storage_config)

tests/unit/borg/test_init.py

@@ -3,8 +3,8 @@ import logging
 from flexmock import flexmock
 
 from borgmatic.borg import init as module
-from ..test_verbosity import insert_logging_mock
 
+from ..test_verbosity import insert_logging_mock
 
 INFO_REPOSITORY_EXISTS_RESPONSE_CODE = 0
 INFO_REPOSITORY_NOT_FOUND_RESPONSE_CODE = 2

tests/unit/borg/test_list.py

@@ -3,57 +3,75 @@ import logging
 from flexmock import flexmock
 
 from borgmatic.borg import list as module
+
 from ..test_verbosity import insert_logging_mock
 
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(check_call_command, **kwargs).once()
-
-
 LIST_COMMAND = ('borg', 'list', 'repo')
 
 
 def test_list_archives_calls_borg_with_parameters():
-    insert_subprocess_mock(LIST_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(LIST_COMMAND, capture_output=False)
 
     module.list_archives(repository='repo', storage_config={})
 
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
-    insert_subprocess_mock(LIST_COMMAND + ('--info',))
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--info',), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
 
     module.list_archives(repository='repo', storage_config={})
 
 
 def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
-    insert_subprocess_mock(LIST_COMMAND + ('--debug', '--show-rc'))
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--debug', '--show-rc'), capture_output=False
+    )
     insert_logging_mock(logging.DEBUG)
 
     module.list_archives(repository='repo', storage_config={})
 
 
-def test_list_archives_with_json_calls_borg_with_json_parameter():
-    insert_subprocess_mock(LIST_COMMAND + ('--json',))
+def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--lock-wait', '5'), capture_output=False
+    )
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(repository='repo', storage_config=storage_config)
+
+
+def test_list_archives_with_archive_calls_borg_with_archive_parameter():
+    storage_config = {}
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo::archive'), capture_output=False
+    )
+
+    module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
 
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
-    insert_subprocess_mock(('borg1',) + LIST_COMMAND[1:])
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1',) + LIST_COMMAND[1:], capture_output=False
+    )
 
     module.list_archives(repository='repo', storage_config={}, local_path='borg1')
 
 
 def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
-    insert_subprocess_mock(LIST_COMMAND + ('--remote-path', 'borg1'))
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+    )
 
     module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
 
 
-def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
-    storage_config = {'lock_wait': 5}
-    insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
+def test_list_archives_with_json_calls_borg_with_json_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--json',), capture_output=True
+    ).and_return('[]')
 
-    module.list_archives(repository='repo', storage_config=storage_config)
+    json_output = module.list_archives(repository='repo', storage_config={}, json=True)
+
+    assert json_output == '[]'

tests/unit/borg/test_prune.py

@@ -4,6 +4,7 @@ from collections import OrderedDict
 from flexmock import flexmock
 
 from borgmatic.borg import prune as module
+
 from ..test_verbosity import insert_logging_mock
 
 

tests/unit/commands/test_borgmatic.py

@@ -1,73 +1,101 @@
-import json
-import sys
-
 from flexmock import flexmock
 
 from borgmatic.commands import borgmatic as module
 
 
-def test_run_commands_handles_multiple_json_outputs_in_array():
-    (
-        flexmock(module)
-        .should_receive('_run_commands_on_repository')
-        .times(3)
-        .replace_with(
-            lambda args, consistency, json_results, local_path, location, remote_path, retention, storage, unexpanded_repository: json_results.append(
-                {"whatever": unexpanded_repository}
-            )
-        )
-    )
-
-    (
-        flexmock(sys.stdout)
-        .should_call("write")
-        .with_args(
-            json.dumps(
-                json.loads(
-                    '''
-                        [
-                            {"whatever": "fake_repo1"},
-                            {"whatever": "fake_repo2"},
-                            {"whatever": "fake_repo3"}
-                        ]
-                    '''
-                )
-            )
-        )
-    )
-
-    module._run_commands(
-        args=flexmock(json=True),
-        consistency=None,
-        local_path=None,
-        location={'repositories': ['fake_repo1', 'fake_repo2', 'fake_repo3']},
-        remote_path=None,
-        retention=None,
-        storage=None,
-    )
-
-
 def test_collect_configuration_run_summary_logs_info_for_success():
-    flexmock(module).should_receive('run_configuration')
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(extract=False, list=False)
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=()))
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
 
-    assert any(log for log in logs if log.levelno == module.logging.INFO)
+    assert all(log for log in logs if log.levelno == module.logging.INFO)
 
 
-def test_collect_configuration_run_summary_logs_critical_for_error():
+def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository')
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(extract=True, list=False, repository='repo')
+
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+
+    assert all(log for log in logs if log.levelno == module.logging.INFO)
+
+
+def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
+        ValueError
+    )
+    args = flexmock(extract=True, list=False, repository='repo')
+
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+
+    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+
+
+def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
+        ValueError
+    )
+    args = flexmock(extract=False, list=True, repository='repo', archive='test')
+
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+
+    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+
+
+def test_collect_configuration_run_summary_logs_info_for_success_with_list():
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(extract=False, list=True, repository='repo', archive=None)
+
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+
+    assert all(log for log in logs if log.levelno == module.logging.INFO)
+
+
+def test_collect_configuration_run_summary_logs_critical_for_parse_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
+    args = flexmock(extract=False, list=False)
+
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+
+    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+
+
+def test_collect_configuration_run_summary_logs_critical_for_run_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository')
     flexmock(module).should_receive('run_configuration').and_raise(ValueError)
+    args = flexmock(extract=False, list=False)
 
-    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=()))
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
 
 
 def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
-    logs = tuple(
-        module.collect_configuration_run_summary_logs(
-            config_filenames=(), args=flexmock(config_paths=())
-        )
-    )
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(config_paths=(), extract=False, list=False)
+
+    logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args))
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+
+
+def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
+    flexmock(module.validate).should_receive('parse_configuration').and_return(
+        {'test.yaml': {}, 'test2.yaml': {}}
+    )
+    flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
+        ['baz']
+    )
+    flexmock(module.sys.stdout).should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    args = flexmock(extract=False, list=False)
+
+    tuple(module.collect_configuration_run_summary_logs(('test.yaml', 'test2.yaml'), args=args))

tests/unit/commands/test_hook.py

@@ -7,7 +7,7 @@ def test_execute_hook_invokes_each_command():
     subprocess = flexmock(module.subprocess)
     subprocess.should_receive('check_call').with_args(':', shell=True).once()
 
-    module.execute_hook([':'], 'config.yaml', 'pre-backup')
+    module.execute_hook([':'], 'config.yaml', 'pre-backup', dry_run=False)
 
 
 def test_execute_hook_with_multiple_commands_invokes_each_command():
@@ -15,8 +15,15 @@ def test_execute_hook_with_multiple_commands_invokes_each_command():
     subprocess.should_receive('check_call').with_args(':', shell=True).once()
     subprocess.should_receive('check_call').with_args('true', shell=True).once()
 
-    module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup')
+    module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup', dry_run=False)
+
+
+def test_execute_hook_with_dry_run_skips_commands():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').never()
+
+    module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup', dry_run=True)
 
 
 def test_execute_hook_with_empty_commands_does_not_raise():
-    module.execute_hook([], 'config.yaml', 'post-backup')
+    module.execute_hook([], 'config.yaml', 'post-backup', dry_run=False)

tests/unit/config/test_collect.py

@@ -1,3 +1,5 @@
+import sys
+
 from flexmock import flexmock
 
 from borgmatic.config import collect as module
@@ -37,7 +39,10 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
-    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml'])
+    flexmock(module.os).should_receive('listdir')
+    flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
+        ['foo.yaml', 'bar', 'baz.yaml']
+    )
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
@@ -56,7 +61,10 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False)
-    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar.yaml~', 'baz.txt'])
+    flexmock(module.os).should_receive('listdir')
+    flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
+        ['foo.yaml', 'bar.yaml~', 'baz.txt']
+    )
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 

tests/unit/config/test_convert.py

@@ -1,12 +1,11 @@
-from collections import defaultdict, OrderedDict, namedtuple
 import os
+from collections import OrderedDict, defaultdict, namedtuple
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import convert as module
 
-
 Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
 
 
@@ -91,7 +90,7 @@ def test_guard_configuration_upgraded_raises_when_only_source_config_present():
     flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
     flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
 
-    with pytest.raises(module.LegacyConfigurationNotUpgraded):
+    with pytest.raises(module.Legacy_configuration_not_upgraded):
         module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
 
 
@@ -117,12 +116,3 @@ def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present
     flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
 
     module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
-
-
-def test_guard_excludes_filename_omitted_raises_when_filename_provided():
-    with pytest.raises(module.LegacyExcludesFilenamePresent):
-        module.guard_excludes_filename_omitted(excludes_filename='/etc/borgmatic/excludes')
-
-
-def test_guard_excludes_filename_omitted_does_not_raise_when_filename_not_provided():
-    module.guard_excludes_filename_omitted(excludes_filename=None)

tests/unit/config/test_legacy.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import legacy as module
 

tests/unit/config/test_validate.py

@@ -90,3 +90,42 @@ def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_
 
 def test_apply_logical_validation_does_not_raise_otherwise():
     module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}})
+
+
+def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
+    module.guard_configuration_contains_repository(
+        repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}}
+    )
+
+
+def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given():
+    module.guard_configuration_contains_repository(
+        repository=None, configurations={'config.yaml': {'location': {'repositories': ['repo']}}}
+    )
+
+
+def test_guard_configuration_contains_repository_errors_when_repository_assumed_to_match_config_twice():
+    with pytest.raises(ValueError):
+        module.guard_configuration_contains_repository(
+            repository=None,
+            configurations={'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}},
+        )
+
+
+def test_guard_configuration_contains_repository_errors_when_repository_missing_from_config():
+    with pytest.raises(ValueError):
+        module.guard_configuration_contains_repository(
+            repository='nope',
+            configurations={'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}},
+        )
+
+
+def test_guard_configuration_contains_repository_errors_when_repository_matches_config_twice():
+    with pytest.raises(ValueError):
+        module.guard_configuration_contains_repository(
+            repository='repo',
+            configurations={
+                'config.yaml': {'location': {'repositories': ['repo', 'repo2']}},
+                'other.yaml': {'location': {'repositories': ['repo']}},
+            },
+        )

tests/unit/test_logger.py

@@ -0,0 +1,74 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic import logger as module
+
+
+@pytest.mark.parametrize('bool_val', (True, 'yes', 'on', '1', 'true', 'True', 1))
+def test_to_bool_parses_true_values(bool_val):
+    assert module.to_bool(bool_val)
+
+
+@pytest.mark.parametrize('bool_val', (False, 'no', 'off', '0', 'false', 'False', 0))
+def test_to_bool_parses_false_values(bool_val):
+    assert not module.to_bool(bool_val)
+
+
+def test_to_bool_passes_none_through():
+    assert module.to_bool(None) is None
+
+
+def test_should_do_markup_respects_no_color_value():
+    assert module.should_do_markup(no_color=True) is False
+
+
+def test_should_do_markup_respects_PY_COLORS_environment_variable():
+    flexmock(module.os.environ).should_receive('get').and_return('True')
+    flexmock(module).should_receive('to_bool').and_return(True)
+
+    assert module.should_do_markup(no_color=False) is True
+
+
+def test_should_do_markup_prefers_no_color_value_to_PY_COLORS():
+    flexmock(module.os.environ).should_receive('get').and_return('True')
+    flexmock(module).should_receive('to_bool').and_return(True)
+
+    assert module.should_do_markup(no_color=True) is False
+
+
+def test_should_do_markup_respects_stdout_tty_value():
+    flexmock(module.os.environ).should_receive('get').and_return(None)
+
+    assert module.should_do_markup(no_color=False) is False
+
+
+def test_should_do_markup_prefers_PY_COLORS_to_stdout_tty_value():
+    flexmock(module.os.environ).should_receive('get').and_return('True')
+    flexmock(module).should_receive('to_bool').and_return(True)
+
+    assert module.should_do_markup(no_color=False) is True
+
+
+@pytest.mark.parametrize('method_name', ('critical', 'error', 'warn', 'info', 'debug'))
+def test_borgmatic_logger_log_method_does_not_raise(method_name):
+    flexmock(module).should_receive('color_text')
+    flexmock(module.logging.Logger).should_receive(method_name)
+
+    getattr(module.Borgmatic_logger('test'), method_name)(msg='hi')
+
+
+def test_borgmatic_logger_handle_does_not_raise():
+    flexmock(module).should_receive('color_text')
+    flexmock(module.logging.Logger).should_receive('handle')
+
+    module.Borgmatic_logger('test').handle(
+        module.logging.makeLogRecord(dict(levelno=module.logging.CRITICAL, msg='hi'))
+    )
+
+
+def test_color_text_does_not_raise():
+    module.color_text(module.colorama.Fore.RED, 'hi')
+
+
+def test_color_text_without_color_does_not_raise():
+    module.color_text(None, 'hi')

tox.ini

@@ -1,27 +1,32 @@
 [tox]
-envlist = py3
+envlist = py35,py36,py37
+skip_missing_interpreters = True
 skipsdist = True
+minversion = 3.10.0
 
 [testenv]
 usedevelop = True
 deps = -rtest_requirements.txt
 whitelist_externals = sh
+install_command =
+    sh scripts/pip {opts} {packages}
 commands =
-    py.test --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end \
-        tests []
-    sh scripts/black
+    pytest
+    py36,py37: black --check .
+    isort --recursive --check-only --settings-path setup.cfg .
     flake8 .
 
 [testenv:black]
 basepython = python3.7
 commands =
-    black --skip-string-normalization --line-length 100 .
+    black .
 
 [testenv:end-to-end]
 deps = -rtest_requirements.txt
 commands =
-    py.test tests/end-to-end []
+    pytest tests/end-to-end
 
-[flake8]
-ignore = E501,W503
-exclude = *.*/*
+[testenv:isort]
+deps = {[testenv]deps}
+commands =
+    isort {posargs:--recursive} --settings-path setup.cfg .