Update docs about MySQL/MariaDB database dumps and restores (#228).

https://yaml.org/spec/1.2/spec.html#id2780069

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.tox

.drone.yml

@@ -1,17 +1,67 @@
-pipeline:
-  build:
-    image: python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
-    pull: true
-    commands:
-      - pip install tox
-      - tox
-      - apk add --no-cache borgbackup
-      - tox -e end-to-end
+---
+kind: pipeline
+name: python-3-5-alpine-3-10
 
-matrix:
-  ALPINE_VERSION:
-    - 3.7
-    - 3.8
-  PYTHON_VERSION:
-    - 3.6
-    - 3.7
+steps:
+- name: build
+  image: python:3.5-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-6-alpine-3-10
+
+steps:
+- name: build
+  image: python:3.6-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-10
+
+steps:
+- name: build
+  image: python:3.7-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-7
+
+steps:
+- name: build
+  image: python:3.7-alpine3.7
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-8-alpine-3-10
+
+steps:
+- name: build
+  image: python:3.8-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: documentation
+
+steps:
+- name: build
+  image: plugins/docker
+  settings:
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    repo: witten/borgmatic-docs
+    dockerfile: docs/Dockerfile
+  when:
+    branch:
+      - master

.eleventy.js

@@ -0,0 +1,42 @@
+const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
+const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
+
+module.exports = function(eleventyConfig) {
+    eleventyConfig.addPlugin(pluginSyntaxHighlight);
+    eleventyConfig.addPlugin(inclusiveLangPlugin);
+
+    let markdownIt = require("markdown-it");
+    let markdownItAnchor = require("markdown-it-anchor");
+    let markdownItReplaceLink = require("markdown-it-replace-link");
+
+    let markdownItOptions = {
+        html: true,
+        breaks: false,
+        linkify: true,
+        replaceLink: function (link, env) {
+            if (process.env.NODE_ENV == "production") {
+                return link;
+            }
+            return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
+        }
+    };
+    let markdownItAnchorOptions = {
+        permalink: true,
+        permalinkClass: "direct-link"
+    };
+
+    eleventyConfig.setLibrary(
+        "md",
+        markdownIt(markdownItOptions)
+            .use(markdownItAnchor, markdownItAnchorOptions)
+            .use(markdownItReplaceLink)
+    );
+
+    return {
+        templateFormats: [
+          "md",
+          "txt"
+        ]
+    }
+};
+

.gitea/issue_template.md

@@ -0,0 +1,31 @@
+#### What I'm trying to do and why
+
+#### Steps to reproduce (if a bug)
+
+Include (sanitized) borgmatic configuration files if applicable.
+
+#### Actual behavior (if a bug)
+
+Include (sanitized) `--verbosity 2` output if applicable.
+
+#### Expected behavior (if a bug)
+
+#### Other notes / implementation ideas
+
+#### Environment
+
+**borgmatic version:** [version here]
+
+Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+
+**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
+
+**Borg version:** [version here]
+
+Use `sudo borg --version`
+
+**Python version:** [version here]
+
+Use `python3 --version`
+
+**operating system and version:** [OS here]

.gitignore

@@ -5,5 +5,7 @@
 .coverage
 .pytest_cache
 .tox
-build
-dist
+__pycache__
+build/
+dist/
+pip-wheel-metadata/

AUTHORS

@@ -7,6 +7,8 @@ Johannes Feichtner: Support for user hooks
 Michele Lazzeri: Custom archive names
 Nick Whyte: Support prefix filtering for archive consistency checks
 newtonne: Read encryption password from external file
-Robin `ypid` Schneider: Support additional options of Borg
+Robin `ypid` Schneider: Support additional options of Borg and add validate-borgmatic-config command
 Scott Squires: Custom archive names
 Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option
+
+Any many others! See the output of "git log".

MANIFEST.in

@@ -1 +1,2 @@
 include borgmatic/config/schema.yaml
+graft sample/systemd

NEWS

@@ -1,3 +1,244 @@
+1.4.9
+ * #228: Database dump hooks for MySQL/MariaDB, so you can easily dump your databases before backups
+   run.
+ * #243: Fix repository does not exist error with "borgmatic extract" when repository is remote.
+
+1.4.8
+ * Monitor backups with Cronhub hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook
+ * Fix Healthchecks/Cronitor hooks to skip actions when the borgmatic "--dry-run" flag is used.
+
+1.4.7
+ * #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
+ * #239: Upgrade your borgmatic configuration to get new options and comments via
+   "generate-borgmatic-config --source". See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration
+
+1.4.6
+ * Verbosity level "-1" for even quieter output: Errors only (#236).
+
+1.4.5
+ * Log to file instead of syslog via command-line "--log-file" flag (#233).
+
+1.4.4
+ * #234: Support for Borg --keep-exclude-tags and --exclude-nodump options.
+
+1.4.3
+ * Monitor backups with Cronitor hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+
+1.4.2
+ * Extract files to a particular directory via "borgmatic extract --destination" flag.
+ * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate
+   "borgmatic restore" action. Any uses of "--restore-path" will continue working.
+
+1.4.1
+ * #229: Restore backed up PostgreSQL databases via "borgmatic restore" action. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/
+ * Documentation on how to develop borgmatic's documentation:
+   https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/#documentation-development
+
+1.4.0
+ * #225: Database dump hooks for PostgreSQL, so you can easily dump your databases before backups
+   run.
+ * #230: Rename "borgmatic list --pattern-from" flag to "--patterns-from" to match Borg.
+
+1.3.26
+ * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
+   (non-checkpoint) archives.
+
+1.3.25
+ * #223: Dead man's switch to detect when backups start failing silently, implemented via
+   healthchecks.io hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+ * Documentation on monitoring and alerting options for borgmatic backups:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/
+ * Automatically rewrite links when developing on documentation locally.
+
+1.3.24
+ * #86: Add "borgmatic list --successful" flag to only list successful (non-checkpoint) archives.
+ * Add a suggestion form to all documentation pages, so users can submit ideas for improving the
+   documentation.
+ * Update documentation link to community Arch Linux borgmatic package.
+
+1.3.23
+ * #174: More detailed error alerting via runtime context available in "on_error" hook.
+
+1.3.22
+ * #144: When backups to one of several repositories fails, keep backing up to the other
+   repositories and report errors afterwards.
+
+1.3.21
+ * #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+
+1.3.20
+ * #205: More robust sample systemd service: boot delay, network dependency, lowered CPU/IO
+   priority, etc.
+ * #221: Fix "borgmatic create --progress" output so that it updates on the console in real-time.
+
+1.3.19
+ * #219: Fix visibility of "borgmatic prune --stats" output.
+
+1.3.18
+ * #220: Fix regression of argument parsing for default actions.
+
+1.3.17
+ * #217: Fix error with "borgmatic check --only" command-line flag with "extract" consistency check.
+
+1.3.16
+ * #210: Support for Borg check --verify-data flag via borgmatic "data" consistency check.
+ * #210: Override configured consistency checks via "borgmatic check --only" command-line flag.
+ * When generating sample configuration with generate-borgmatic-config, add a space after each "#"
+   comment indicator.
+
+1.3.15
+ * #208: Fix for traceback when the "checks" option has an empty value.
+ * #209: Bypass Borg error about a moved repository via "relocated_repo_access_is_ok" option in
+   borgmatic storage configuration section.
+ * #213: Reorder arguments passed to Borg to fix duplicate directories when using Borg patterns.
+ * #214: Fix for hook erroring with exit code 1 not being interpreted as an error.
+
+1.3.14
+ * #204: Do not treat Borg warnings (exit code 1) as failures.
+ * When validating configuration files, require strings instead of allowing any scalar type.
+
+1.3.13
+ * #199: Add note to documentation about using spaces instead of tabs for indentation, as YAML does
+   not allow tabs.
+ * #203: Fix compatibility with ruamel.yaml 0.16.x.
+ * If a "prefix" option in borgmatic's configuration has an empty value (blank or ""), then disable
+   default prefix.
+
+1.3.12
+ * Only log to syslog when run from a non-interactive console (e.g. a cron job).
+ * Remove unicode byte order mark from syslog output so it doesn't show up as a literal in rsyslog
+   output. See discussion on #197.
+
+1.3.11
+ * #193: Pass through several "borg list" and "borg info" flags like --short, --format, --sort-by,
+   --first, --last, etc. via borgmatic command-line flags.
+ * Add borgmatic info --repository and --archive command-line flags to display info for individual
+   repositories or archives.
+ * Support for Borg --noatime, --noctime, and --nobirthtime flags via corresponding options in
+   borgmatic configuration location section.
+
+1.3.10
+ * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.
+
+1.3.9
+ * #195: Switch to command-line actions as more traditional sub-commands, e.g. "borgmatic create",
+   "borgmatic prune", etc. However, the classic dashed options like "--create" still work!
+
+1.3.8
+ * #191: Disable console color via "color" option in borgmatic configuration output section.
+
+1.3.7
+ * #196: Fix for unclear error message for invalid YAML merge include.
+ * #197: Don't color syslog output.
+ * Change default syslog verbosity to show errors only.
+
+1.3.6
+ * #53: Log to syslog in addition to existing console logging. Add --syslog-verbosity flag to
+   customize the log level. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/
+ * #178: Look for .yml configuration file extension in addition to .yaml.
+ * #189: Set umask used when executing hooks via "umask" option in borgmatic hooks section.
+ * Remove Python cache files before each Tox run.
+ * Add #borgmatic Freenode IRC channel to documentation.
+ * Add Borg/borgmatic hosting providers section to documentation.
+ * Add files for building documentation into a Docker image for web serving.
+ * Upgrade project build server from Drone 0.8 to 1.1.
+ * Build borgmatic documentation during continuous integration.
+ * We're nearly at 500 ★s on GitHub. We can do this!
+
+1.3.5
+ * #153: Support for various Borg directory environment variables (BORG_CONFIG_DIR, BORG_CACHE_DIR,
+   etc.) via options in borgmatic's storage configuration.
+ * #177: Fix for regression with missing verbose log entries.
+
+1.3.4
+ * Part of #125: Color borgmatic (but not Borg) output when using an interactive terminal.
+ * #166: Run tests for all installed versions of Python.
+ * #168: Update README with continuous integration badge.
+ * #169: Automatically sort Python imports in code.
+ * Document installing borgmatic with pip install --user instead of a system Python install.
+ * Get more reproducible builds by pinning the versions of pip and tox used to run tests.
+ * Factor out build/test configuration from tox.ini file.
+
+1.3.3
+ * Add validate-borgmatic-config command, useful for validating borgmatic config generated by
+   configuration management or even edited by hand.
+
+1.3.2
+ * #160: Fix for hooks executing when using --dry-run. Now hooks are skipped during a dry run.
+
+1.3.1
+ * #155: Fix for invalid JSON output when using multiple borgmatic configuration files.
+ * #157: Fix for seemingly random filename ordering when running through a directory of
+   configuration files.
+ * Fix for empty JSON output when using --create --json.
+ * Now capturing Borg output only when --json flag is used. Previously, borgmatic delayed Borg
+   output even without the --json flag.
+
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
+1.2.18
+ * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
+   borgmatic's location section.
+
+1.2.17
+ * #140: List the files within an archive via --list --archive option.
+
+1.2.16
+ * #119: Include a sample borgmatic configuration file in the documentation.
+ * #123: Support for Borg archive restoration via borgmatic --extract command-line flag.
+ * Refactor documentation into multiple separate pages for clarity and findability.
+ * Organize options within command-line help into logical groups.
+ * Exclude tests from distribution packages.
+
+1.2.15
+ * #127: Remove date echo from schema example, as it's not a substitute for real logging.
+ * #132: Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to
+   confusing behavior.
+ * #136: Handle and format validation errors raised during argument parsing.
+ * #138: Allow use of --stats flag when --create or --prune flags are implied.
+
+1.2.14
+ * #103: When generating sample configuration with generate-borgmatic-config, document the defaults
+   for each option.
+ * #116: When running multiple configuration files, attempt all configuration files even if one of
+   them errors. Log a summary of results at the end.
+ * Add borgmatic --version command-line flag to get the current installed version number.
+
+1.2.13
+ * #100: Support for --stats command-line flag independent of --verbosity.
+ * #117: With borgmatic --init command-line flag, proceed without erroring if a repository already
+   exists.
+
+1.2.12
+ * #110: Support for Borg repository initialization via borgmatic --init command-line flag.
+ * #111: Update Borg create --filter values so a dry run lists files to back up.
+ * #113: Update README with link to a new/forked Docker image.
+ * Prevent deprecated --excludes command-line option from being used.
+ * Refactor README a bit to flow better for first-time users.
+ * Update README with a few additional borgmatic packages (Debian and Ubuntu).
+
+1.2.11
+ * #108: Support for Borg create --progress via borgmatic command-line flag.
+
+1.2.10
+ * #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
+   storage section.
+
+1.2.9
+ * #102: Fix for syntax error that occurred in Python 3.5 and below.
+ * Make automated tests support running in Python 3.5.
+
 1.2.8
  * #73: Enable consistency checks for only certain repositories via "check_repositories" option in
    borgmatic's consistency configuration. Handy for large repositories that take forever to check.

README.md

@@ -1,17 +1,20 @@
 ---
 title: borgmatic
-permalink: borgmatic/index.html
+permalink: index.html
 ---
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
+
 ## Overview
 
-<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" width="150px" style="float: right; padding-left: 1em;">
+<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
 
-borgmatic is a simple Python wrapper script for the
-[Borg](https://www.borgbackup.org/) backup software that initiates a backup,
-prunes any old backups according to a retention policy, and validates backups
-for consistency. The script supports specifying your settings in a declarative
-configuration file rather than having to put them all on the command-line, and
-handles common errors.
+borgmatic is simple, configuration-driven backup software for servers and
+workstations. Backup all of your machines from the command-line or scheduled
+jobs. No GUI required. Built atop [Borg Backup](https://www.borgbackup.org/),
+borgmatic initiates a backup, prunes any old backups according to a retention
+policy, and validates backups for consistency. borgmatic supports specifying
+your settings in a declarative configuration file, rather than having to put
+them all on the command-line, and handles common errors.
 
 Here's an example config file:
 
@@ -38,10 +41,18 @@ retention:
     keep_monthly: 6
 
 consistency:
-    # List of consistency checks to run: "repository", "archives", or both.
+    # List of consistency checks to run: "repository", "archives", etc.
     checks:
         - repository
         - archives
+
+hooks:
+    # Preparation scripts to run, databases to dump, and monitoring to perform.
+    before_backup:
+        - prepare-for-backup.sh
+    postgresql_databases:
+        - name: users
+    healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
 ```
 
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
@@ -54,305 +65,37 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 <script src="https://asciinema.org/a/203761.js" id="asciicast-203761" async></script>
 
 
-## Installation
+## How-to guides
 
-To get up and running, first [install
-Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
-least version 1.1. Then, follow the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host.
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) ⬅ *Start here!*
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
+ * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 
-Note that if you plan to run borgmatic on a schedule with cron, and you
-encrypt your Borg repository with a passphrase instead of a key file, you'll
-either need to set the borgmatic `encryption_passphrase` configuration
-variable or set the `BORG_PASSPHRASE` environment variable. See the
-[repository encryption
-section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
-of the Quick Start for more info.
 
-Alternatively, the passphrase can be specified programatically by setting
-either the borgmatic `encryption_passcommand` configuration variable or the
-`BORG_PASSCOMMAND` environment variable. See the [Borg Security
-FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
-for more info.
+## Reference guides
 
-If the repository is on a remote host, make sure that your local root user has
-key-based ssh access to the desired user account on the remote host.
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)
 
-To install borgmatic, run the following command to download and install it:
 
-```bash
-sudo pip3 install --upgrade borgmatic
-```
+## Hosting providers
 
-Note that your pip binary may have a different name than "pip3". Make sure
-you're using Python 3, as borgmatic does not support Python 2.
-
-### Other ways to install
-
- * [A borgmatic Docker image](https://hub.docker.com/r/b3vis/borgmatic/) based
-   on Alpine Linux.
- * [Another borgmatic Docker image](https://hub.docker.com/r/coaxial/borgmatic/) based
-   on Alpine Linux, updated automatically whenever the Alpine image updates.
- * [A borgmatic package for
-   Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic).
- * [A borgmatic package for Arch
-   Linux](https://aur.archlinux.org/packages/borgmatic/).
- * [A borgmatic package for OpenBSD](http://ports.su/sysutils/borgmatic).
-<br><br>
-
-## Configuration
-
-After you install borgmatic, generate a sample configuration file:
-
-```bash
-sudo generate-borgmatic-config
-```
-
-If that command is not found, then it may be installed in a location that's
-not in your system `PATH`. Try looking in `/usr/local/bin/`.
-
-This generates a sample configuration file at /etc/borgmatic/config.yaml (by
-default). You should edit the file to suit your needs, as the values are just
-representative. All fields are optional except where indicated, so feel free
-to ignore anything you don't need.
-
-You can also have a look at the [full configuration
-schema](https://projects.torsion.org/witten/borgmatic/src/master/borgmatic/config/schema.yaml)
-for the authoritative set of all configuration options. This is handy if
-borgmatic has added new options since you originally created your
-configuration file.
-
-
-### Multiple configuration files
-
-A more advanced usage is to create multiple separate configuration files and
-place each one in an /etc/borgmatic.d directory. For instance:
-
-```bash
-sudo mkdir /etc/borgmatic.d
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
-```
-
-With this approach, you can have entirely different backup policies for
-different applications on your system. For instance, you may want one backup
-configuration for your database data directory, and a different configuration
-for your user home directories.
-
-When you set up multiple configuration files like this, borgmatic will run
-each one in turn from a single borgmatic invocation. This includes, by
-default, the traditional /etc/borgmatic/config.yaml as well.
-
-And if you need even more customizability, you can specify alternate
-configuration paths on the command-line with borgmatic's `--config` option.
-See `borgmatic --help` for more information.
-
-
-### Hooks
-
-If you find yourself performing prepraration tasks before your backup runs, or
-cleanup work afterwards, borgmatic hooks may be of interest. They're simply
-shell commands that borgmatic executes for you at various points, and they're
-configured in the `hooks` section of your configuration file.
-
-For instance, you can specify `before_backup` hooks to dump a database to file
-before backing it up, and specify `after_backup` hooks to delete the temporary
-file afterwards.
-
-borgmatic hooks run once per configuration file. `before_backup` hooks run
-prior to backups of all repositories. `after_backup` hooks run afterwards, but
-not if an error occurs in a previous hook or in the backups themselves. And
-borgmatic runs `on_error` hooks if an error occurs.
-
-An important security note about hooks: borgmatic executes all hook commands
-with the user permissions of borgmatic itself. So to prevent potential shell
-injection or privilege escalation, do not forget to set secure permissions
-(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
-
-See the sample generated configuration file mentioned above for specifics
-about hook configuration syntax.
-
-
-## Upgrading
-
-In general, all you should need to do to upgrade borgmatic is run the
-following:
-
-```bash
-sudo pip3 install --upgrade borgmatic
-```
-
-However, see below about special cases.
-
-
-### Upgrading from borgmatic 1.0.x
-
-borgmatic changed its configuration file format in version 1.1.0 from
-INI-style to YAML. This better supports validation, and has a more natural way
-to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic.
-
-As of version 1.1.0, borgmatic no longer supports Python 2. If you were
-already running borgmatic with Python 3, then you can simply upgrade borgmatic
-in-place:
-
-```bash
-sudo pip3 install --upgrade borgmatic
-```
-
-But if you were running borgmatic with Python 2, uninstall and reinstall instead:
-
-```bash
-sudo pip uninstall borgmatic
-sudo pip3 install borgmatic
-```
-
-The pip binary names for different versions of Python can differ, so the above
-commands may need some tweaking to work on your machine.
-
-
-Once borgmatic is upgraded, run:
-
-```bash
-sudo upgrade-borgmatic-config
-```
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and
-excludes files. The new version of borgmatic will consume the YAML
-configuration file instead of the old one.
-
-
-### Upgrading from atticmatic
-
-You can ignore this section if you're not an atticmatic user (the former name
-of borgmatic).
-
-borgmatic only supports Borg now and no longer supports Attic. So if you're
-an Attic user, consider switching to Borg. See the [Borg upgrade
-command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
-for more information. Then, follow the instructions above about setting up
-your borgmatic configuration files.
-
-If you were already using Borg with atticmatic, then you can easily upgrade
-from atticmatic to borgmatic. Simply run the following commands:
-
-```bash
-sudo pip3 uninstall atticmatic
-sudo pip3 install borgmatic
-```
-
-That's it! borgmatic will continue using your /etc/borgmatic configuration
-files.
-
-
-## Usage
-
-You can run borgmatic and start a backup simply by invoking it without
-arguments:
-
-```bash
-borgmatic
-```
-
-This will also prune any old backups as per the configured retention policy,
-and check backups for consistency problems due to things like file damage.
-
-If you'd like to see the available command-line arguments, view the help:
-
-```bash
-borgmatic --help
-```
-
-Note that borgmatic prunes archives *before* creating an archive, so as to
-free up space for archiving. This means that when a borgmatic run finishes,
-there may still be prune-able archives. Not to worry, as they will get cleaned
-up at the start of the next run.
-
-### Verbosity
-
-By default, the backup will proceed silently except in the case of errors. But
-if you'd like to to get additional information about the progress of the
-backup as it proceeds, use the verbosity option:
-
-```bash
-borgmatic --verbosity 1
-```
-
-Or, for even more progress spew:
-
-```bash
-borgmatic --verbosity 2
-```
-
-### À la carte
-
-If you want to run borgmatic with only pruning, creating, or checking enabled,
-the following optional flags are available:
-
-```bash
-borgmatic --prune
-borgmatic --create
-borgmatic --check
-```
-
-You can run with only one of these flags provided, or you can mix and match
-any number of them. This supports use cases like running consistency checks
-from a different cron job with a different frequency, or running pruning with
-a different verbosity level.
-
-Additionally, borgmatic provides convenient flags for Borg's
-[list](https://borgbackup.readthedocs.io/en/stable/usage/list.html) and
-[info](https://borgbackup.readthedocs.io/en/stable/usage/info.html)
-functionality:
-
-
-```bash
-borgmatic --list
-borgmatic --info
-```
-
-You can include an optional `--json` flag with `--create`, `--list`, or
-`--info` to get the output formatted as JSON.
-
-
-## Autopilot
-
-If you want to run borgmatic automatically, say once a day, the you can
-configure a job runner to invoke it periodically.
-
-### cron
-
-If you're using cron, download the [sample cron
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic).
-Then, from the directory where you downloaded it:
-
-```bash
-sudo mv borgmatic /etc/cron.d/borgmatic
-sudo chmod +x /etc/cron.d/borgmatic
-```
-
-You can modify the cron file if you'd like to run borgmatic more or less frequently.
-
-### systemd
-
-If you're using systemd instead of cron to run jobs, download the [sample
-systemd service
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.service)
-and the [sample systemd timer
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.timer).
-Then, from the directory where you downloaded them:
-
-```bash
-sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
-sudo systemctl enable borgmatic.timer
-sudo systemctl start borgmatic.timer
-```
-
-Feel free to modify the timer file based on how frequently you'd like
-borgmatic to run.
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
 
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
 
 ## Support and contributing
 
@@ -364,6 +107,11 @@ create a new issue or comment on an issue, you'll need to [login
 first](https://projects.torsion.org/user/login). Note that you can login with
 an existing GitHub account if you prefer.
 
+If you'd like to chat with borgmatic developers or users, head on over to the
+`#borgmatic` IRC channel on Freenode, either via <a
+href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
+native <a href="irc://chat.freenode.net:6697">IRC client</a>.
+
 Other questions or comments? Contact <mailto:witten@torsion.org>.
 
 
@@ -375,130 +123,6 @@ or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
 to discuss your idea. We also accept Pull Requests on GitHub, if that's more
 your thing. In general, contributions are very welcome. We don't bite! 
 
-
-### Code style
-
-Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
-the following deviations from it:
-
- * For strings, prefer single quotes over double quotes.
- * Limit all lines to a maximum of 100 characters.
- * Use trailing commas within multiline values or argument lists.
- * For multiline constructs, put opening and closing delimeters on lines
-   separate from their contents.
- * Within multiline constructs, use standard four-space indentation. Don't align
-   indentation with an opening delimeter.
-
-borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
-formatter and [Flake8](http://flake8.pycqa.org/en/latest/) code checker, so
-certain code style requirements will be enforced when running automated tests.
-See the Black and Flake8 documentation for more information.
-
-
-### Development
-
-To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
-
-```bash
-git clone https://projects.torsion.org/witten/borgmatic.git
-```
-
-Or:
-
-```bash
-git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git
-```
-
-Then, install borgmatic
-"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)"
-so that you can easily run borgmatic commands while you're hacking on them to
-make sure your changes work.
-
-```bash
-cd borgmatic/
-pip3 install --editable --user .
-```
-
-Note that this will typically install the borgmatic commands into
-`~/.local/bin`, which may or may not be on your PATH. There are other ways to
-install borgmatic editable as well, for instance into the system Python
-install (so without `--user`, as root), or even into a
-[virtualenv](https://virtualenv.pypa.io/en/stable/). How or where you install
-borgmatic is up to you, but generally an editable install makes development
-and testing easier.
-
-
-### Running tests
-
-Assuming you've cloned the borgmatic source code as described above, and
-you're in the `borgmatic/` working copy, install tox, which is used for
-setting up testing environments:
-
-```bash
-sudo pip3 install tox
-```
-
-Finally, to actually run tests, run:
-
-```bash
-cd borgmatic
-tox
-```
-
-Note that while running borgmatic itself only requires Python 3+, running
-borgmatic's tests require Python 3.6+.
-
-If when running tests, you get an error from the
-[Black](https://black.readthedocs.io/en/stable/) code formatter about files
-that would be reformatted, you can ask Black to format them for you via the
-following:
-
-```bash
-tox -e black
-```
-
-### End-to-end tests
-
-borgmatic additionally includes some end-to-end tests that integration test
-with Borg for a few representative scenarios. These tests don't run by default
-because they're relatively slow and depend on Borg. If you would like to run
-them:
-
-```bash
-tox -e end-to-end
-```
-
-## Troubleshooting
-
-### Broken pipe with remote repository
-
-When running borgmatic on a large remote repository, you may receive errors
-like the following, particularly while "borg check" is validating backups for
-consistency:
-
-```text
-    Write failed: Broken pipe
-    borg: Error: Connection closed by remote host
-```
-
-This error can be caused by an ssh timeout, which you can rectify by adding
-the following to the `~/.ssh/config` file on the client:
-
-```text
-    Host *
-        ServerAliveInterval 120
-```
-
-This should make the client keep the connection alive while validating
-backups.
-
-
-### libyaml compilation errors
-
-borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
-use a C YAML library (libyaml) if present. But if it's not installed, then
-when installing or upgrading borgmatic, you may see errors about compiling the
-YAML library. If so, not to worry. borgmatic should install and function
-correctly even without the C YAML library. And borgmatic won't be any faster
-with the C library present, so you don't need to go out of your way to install
-it.
+Also, please check out the [borgmatic development
+how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
+info on cloning source code, running tests, etc.

borgmatic/borg/check.py

@@ -1,9 +1,7 @@
 import logging
-import os
-import subprocess
 
 from borgmatic.borg import extract
-
+from borgmatic.execute import execute_command
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
@@ -12,9 +10,10 @@ DEFAULT_PREFIX = '{hostname}-'
 logger = logging.getLogger(__name__)
 
 
-def _parse_checks(consistency_config):
+def _parse_checks(consistency_config, only_checks=None):
     '''
-    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
+    Given a consistency config with a "checks" list, and an optional list of override checks,
+    transform them a tuple of named checks to run.
 
     For example, given a retention config of:
 
@@ -24,16 +23,21 @@ def _parse_checks(consistency_config):
 
         ('repository', 'archives')
 
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If the checks value
+    is the string "disabled", return an empty tuple, meaning that no checks should be run.
+
+    If the "data" option is present, then make sure the "archives" option is included as well.
     '''
-    checks = consistency_config.get('checks', [])
+    checks = [
+        check.lower() for check in (only_checks or consistency_config.get('checks', []) or [])
+    ]
     if checks == ['disabled']:
         return ()
 
-    return (
-        tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
-    )
+    if 'data' in checks and 'archives' not in checks:
+        checks.append('archives')
+
+    return tuple(check for check in checks if check not in ('disabled', '')) or DEFAULT_CHECKS
 
 
 def _make_check_flags(checks, check_last=None, prefix=None):
@@ -57,7 +61,7 @@ def _make_check_flags(checks, check_last=None, prefix=None):
     '''
     if 'archives' in checks:
         last_flags = ('--last', str(check_last)) if check_last else ()
-        prefix_flags = ('--prefix', prefix) if prefix else ('--prefix', DEFAULT_PREFIX)
+        prefix_flags = ('--prefix', prefix) if prefix else ()
     else:
         last_flags = ()
         prefix_flags = ()
@@ -70,30 +74,37 @@ def _make_check_flags(checks, check_last=None, prefix=None):
                 'Ignoring consistency prefix option, as "archives" is not in consistency checks.'
             )
 
+    common_flags = last_flags + prefix_flags + (('--verify-data',) if 'data' in checks else ())
+
     if set(DEFAULT_CHECKS).issubset(set(checks)):
-        return last_flags + prefix_flags
+        return common_flags
 
     return (
         tuple('--{}-only'.format(check) for check in checks if check in DEFAULT_CHECKS)
-        + last_flags
-        + prefix_flags
+        + common_flags
     )
 
 
 def check_archives(
-    repository, storage_config, consistency_config, local_path='borg', remote_path=None
+    repository,
+    storage_config,
+    consistency_config,
+    local_path='borg',
+    remote_path=None,
+    only_checks=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, a consistency config dict,
-    and a local/remote commands to run, check the contained Borg archives for consistency.
+    local/remote commands to run, and an optional list of checks to use instead of configured
+    checks, check the contained Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
-    checks = _parse_checks(consistency_config)
+    checks = _parse_checks(consistency_config, only_checks)
     check_last = consistency_config.get('check_last', None)
     lock_wait = None
 
-    if set(checks).intersection(set(DEFAULT_CHECKS)):
+    if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
         remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
         lock_wait = storage_config.get('lock_wait', None)
         lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
@@ -104,21 +115,18 @@ def check_archives(
         if logger.isEnabledFor(logging.DEBUG):
             verbosity_flags = ('--debug', '--show-rc')
 
-        prefix = consistency_config.get('prefix')
+        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
 
         full_command = (
-            (local_path, 'check', repository)
+            (local_path, 'check')
             + _make_check_flags(checks, check_last, prefix)
             + remote_path_flags
             + lock_wait_flags
             + verbosity_flags
+            + (repository,)
         )
 
-        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-        stdout = None if verbosity_flags else open(os.devnull, 'w')
-
-        logger.debug(' '.join(full_command))
-        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+        execute_command(full_command)
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -2,27 +2,13 @@ import glob
 import itertools
 import logging
 import os
-import subprocess
 import tempfile
 
+from borgmatic.execute import execute_command, execute_command_without_capture
 
 logger = logging.getLogger(__name__)
 
 
-def initialize_environment(storage_config):
-    passcommand = storage_config.get('encryption_passcommand')
-    if passcommand:
-        os.environ['BORG_PASSCOMMAND'] = passcommand
-
-    passphrase = storage_config.get('encryption_passphrase')
-    if passphrase:
-        os.environ['BORG_PASSPHRASE'] = passphrase
-
-    ssh_command = storage_config.get('ssh_command')
-    if ssh_command:
-        os.environ['BORG_RSH'] = ssh_command
-
-
 def _expand_directory(directory):
     '''
     Given a directory path, expand any tilde (representing a user's home directory) and any globs
@@ -46,6 +32,17 @@ def _expand_directories(directories):
     )
 
 
+def _expand_home_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
+    Return the results as a tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(os.path.expanduser(directory) for directory in directories)
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -63,8 +60,8 @@ def _write_pattern_file(patterns=None):
 
 def _make_pattern_flags(location_config, pattern_filename=None):
     '''
-    Given a location config dict with a potential pattern_from option, and a filename containing any
-    additional patterns, return the corresponding Borg flags for those files as a tuple.
+    Given a location config dict with a potential patterns_from option, and a filename containing
+    any additional patterns, return the corresponding Borg flags for those files as a tuple.
     '''
     pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
         (pattern_filename,) if pattern_filename else ()
@@ -93,8 +90,32 @@ def _make_exclude_flags(location_config, exclude_filename=None):
     caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
     if_present = location_config.get('exclude_if_present')
     if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+    keep_exclude_tags_flags = (
+        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
+    )
+    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
 
-    return exclude_from_flags + caches_flag + if_present_flags
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
+    )
+
+
+BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
+
+
+def borgmatic_source_directories():
+    '''
+    Return a list of borgmatic-specific source directories used for state like database backups.
+    '''
+    return (
+        [BORGMATIC_SOURCE_DIRECTORY]
+        if os.path.exists(os.path.expanduser(BORGMATIC_SOURCE_DIRECTORY))
+        else []
+    )
 
 
 def create_archive(
@@ -104,17 +125,24 @@ def create_archive(
     storage_config,
     local_path='borg',
     remote_path=None,
+    progress=False,
+    stats=False,
     json=False,
 ):
     '''
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
-    storage config dict, create a Borg archive.
+    storage config dict, create a Borg archive and return Borg's JSON output (if any).
     '''
-    sources = _expand_directories(location_config['source_directories'])
+    sources = _expand_directories(
+        location_config['source_directories'] + borgmatic_source_directories()
+    )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
-    exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
+    exclude_file = _write_pattern_file(
+        _expand_home_directories(location_config.get('exclude_patterns'))
+    )
     checkpoint_interval = storage_config.get('checkpoint_interval', None)
+    chunker_params = storage_config.get('chunker_params', None)
     compression = storage_config.get('compression', None)
     remote_rate_limit = storage_config.get('remote_rate_limit', None)
     umask = storage_config.get('umask', None)
@@ -124,33 +152,54 @@ def create_archive(
     archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
 
     full_command = (
-        (
-            local_path,
-            'create',
-            '{repository}::{archive_name_format}'.format(
-                repository=repository, archive_name_format=archive_name_format
-            ),
-        )
-        + sources
+        (local_path, 'create')
         + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
         + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--noatime',) if location_config.get('atime') is False else ())
+        + (('--noctime',) if location_config.get('ctime') is False else ())
+        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
         + (('--read-special',) if location_config.get('read_special') else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--list', '--filter', 'AME') if logger.isEnabledFor(logging.INFO) else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--list', '--filter', 'AME-') if logger.isEnabledFor(logging.INFO) and not json else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (
+            ('--stats',)
+            if not dry_run and (logger.isEnabledFor(logging.INFO) or stats) and not json
+            else ()
+        )
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
         + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
         + (('--json',) if json else ())
+        + (
+            '{repository}::{archive_name_format}'.format(
+                repository=repository, archive_name_format=archive_name_format
+            ),
+        )
+        + sources
     )
 
-    logger.debug(' '.join(full_command))
-    subprocess.check_call(full_command)
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(full_command)
+        return
+
+    if json:
+        output_log_level = None
+    elif stats:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    return execute_command(full_command, output_log_level)

borgmatic/borg/environment.py

@@ -0,0 +1,31 @@
+import os
+
+OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'borg_base_directory': 'BORG_BASE_DIR',
+    'borg_config_directory': 'BORG_CONFIG_DIR',
+    'borg_cache_directory': 'BORG_CACHE_DIR',
+    'borg_security_directory': 'BORG_SECURITY_DIR',
+    'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
+    'ssh_command': 'BORG_RSH',
+}
+
+DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
+    'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
+}
+
+
+def initialize(storage_config):
+    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name)
+        if value:
+            os.environ[environment_variable_name] = value
+
+    for (
+        option_name,
+        environment_variable_name,
+    ) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name, False)
+        os.environ[environment_variable_name] = 'yes' if value else 'no'

borgmatic/borg/extract.py

@@ -1,15 +1,15 @@
 import logging
-import sys
-import subprocess
+import os
 
+from borgmatic.execute import execute_command, execute_command_without_capture
 
 logger = logging.getLogger(__name__)
 
 
 def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg', remote_path=None):
     '''
-    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
-    the dry-run.
+    Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
+    dry-run.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
@@ -20,33 +20,83 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         verbosity_flags = ('--info',)
 
     full_list_command = (
-        (local_path, 'list', '--short', repository)
+        (local_path, 'list', '--short')
         + remote_path_flags
         + lock_wait_flags
         + verbosity_flags
+        + (repository,)
     )
 
-    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
+    list_output = execute_command(full_list_command, output_log_level=None)
 
-    last_archive_name = list_output.strip().split('\n')[-1]
-    if not last_archive_name:
+    try:
+        last_archive_name = list_output.strip().splitlines()[-1]
+    except IndexError:
         return
 
     list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
     full_extract_command = (
-        (
-            local_path,
-            'extract',
-            '--dry-run',
-            '{repository}::{last_archive_name}'.format(
-                repository=repository, last_archive_name=last_archive_name
-            ),
-        )
+        (local_path, 'extract', '--dry-run')
         + remote_path_flags
         + lock_wait_flags
         + verbosity_flags
         + list_flag
+        + (
+            '{repository}::{last_archive_name}'.format(
+                repository=repository, last_archive_name=last_archive_name
+            ),
+        )
     )
 
-    logger.debug(' '.join(full_extract_command))
-    subprocess.check_call(full_extract_command)
+    execute_command(full_extract_command, working_directory=None, error_on_warnings=True)
+
+
+def extract_archive(
+    dry_run,
+    repository,
+    archive,
+    paths,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    destination_path=None,
+    progress=False,
+    error_on_warnings=True,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
+    restore from the archive, location/storage configuration dicts, optional local and remote Borg
+    paths, and an optional destination path to extract to, extract the archive into the current
+    directory.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'extract')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+        + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
+        + (tuple(paths) if paths else ())
+    )
+
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    if progress:
+        execute_command_without_capture(
+            full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
+        )
+        return
+
+    # Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
+    # the archive!
+    execute_command(
+        full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
+    )

borgmatic/borg/flags.py

@@ -0,0 +1,31 @@
+import itertools
+
+
+def make_flags(name, value):
+    '''
+    Given a flag name and its value, return it formatted as Borg-compatible flags.
+    '''
+    if not value:
+        return ()
+
+    flag = '--{}'.format(name.replace('_', '-'))
+
+    if value is True:
+        return (flag,)
+
+    return (flag, str(value))
+
+
+def make_flags_from_arguments(arguments, excludes=()):
+    '''
+    Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
+    list of named arguments to exclude, generate and return the corresponding Borg command-line
+    flags as a tuple.
+    '''
+    return tuple(
+        itertools.chain.from_iterable(
+            make_flags(name, value=getattr(arguments, name))
+            for name in sorted(vars(arguments))
+            if name not in excludes and not name.startswith('_')
+        )
+    )

borgmatic/borg/info.py

@@ -1,29 +1,43 @@
 import logging
-import subprocess
 
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
 def display_archives_info(
-    repository, storage_config, local_path='borg', remote_path=None, json=False
+    repository, storage_config, info_arguments, local_path='borg', remote_path=None
 ):
     '''
-    Given a local or remote repository path, and a storage config dict,
-    display summary information for Borg archives in the repository.
+    Given a local or remote repository path, a storage config dict, and the arguments to the info
+    action, display summary information for Borg archives in the repository or return JSON summary
+    information.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'info', repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--json',) if json else ())
+        (local_path, 'info')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not info_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not info_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
+        + (
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
+        )
     )
 
-    logger.debug(' '.join(full_command))
-
-    output = subprocess.check_output(full_command)
-    return output.decode() if output is not None else None
+    return execute_command(
+        full_command, output_log_level=None if info_arguments.json else logging.WARNING
+    )

borgmatic/borg/init.py

@@ -0,0 +1,48 @@
+import logging
+import subprocess
+
+from borgmatic.execute import execute_command, execute_command_without_capture
+
+logger = logging.getLogger(__name__)
+
+
+INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
+
+
+def initialize_repository(
+    repository,
+    encryption_mode,
+    append_only=None,
+    storage_quota=None,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a Borg encryption mode, whether the repository should
+    be append-only, and the storage quota to use, initialize the repository. If the repository
+    already exists, then log and skip initialization.
+    '''
+    info_command = (local_path, 'info', repository)
+    logger.debug(' '.join(info_command))
+
+    try:
+        execute_command(info_command, output_log_level=None)
+        logger.info('Repository already exists. Skipping initialization.')
+        return
+    except subprocess.CalledProcessError as error:
+        if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
+            raise
+
+    init_command = (
+        (local_path, 'init')
+        + (('--encryption', encryption_mode) if encryption_mode else ())
+        + (('--append-only',) if append_only else ())
+        + (('--storage-quota', storage_quota) if storage_quota else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (repository,)
+    )
+
+    # Don't use execute_command() here because it doesn't support interactive prompts.
+    execute_command_without_capture(init_command)

borgmatic/borg/list.py

@@ -1,26 +1,50 @@
 import logging
-import subprocess
 
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def list_archives(repository, storage_config, local_path='borg', remote_path=None, json=False):
+# A hack to convince Borg to exclude archives ending in ".checkpoint". This assumes that a
+# non-checkpoint archive name ends in a digit (e.g. from a timestamp).
+BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
+
+
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
-    Given a local or remote repository path, and a storage config dict,
-    list Borg archives in the repository.
+    Given a local or remote repository path, a storage config dict, and the arguments to the list
+    action, display the output of listing Borg archives in the repository or return JSON output. Or,
+    if an archive name is given, listing the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
+    if list_arguments.successful:
+        list_arguments.glob_archives = BORG_EXCLUDE_CHECKPOINTS_GLOB
 
     full_command = (
-        (local_path, 'list', repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--json',) if json else ())
+        (local_path, 'list')
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(
+            list_arguments, excludes=('repository', 'archive', 'successful')
+        )
+        + (
+            '::'.join((repository, list_arguments.archive))
+            if list_arguments.archive
+            else repository,
+        )
     )
-    logger.debug(' '.join(full_command))
 
-    output = subprocess.check_output(full_command)
-    return output.decode() if output is not None else None
+    return execute_command(
+        full_command, output_log_level=None if list_arguments.json else logging.WARNING
+    )

borgmatic/borg/prune.py

@@ -1,6 +1,6 @@
 import logging
-import subprocess
 
+from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -21,17 +21,26 @@ def _make_prune_flags(retention_config):
             ('--keep-monthly', '6'),
         )
     '''
-    if not retention_config.get('prefix'):
-        retention_config['prefix'] = '{hostname}-'
+    config = retention_config.copy()
+
+    if 'prefix' not in config:
+        config['prefix'] = '{hostname}-'
+    elif not config['prefix']:
+        config.pop('prefix')
 
     return (
-        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
-        for option_name, value in retention_config.items()
+        ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
     )
 
 
 def prune_archives(
-    dry_run, repository, storage_config, retention_config, local_path='borg', remote_path=None
+    dry_run,
+    repository,
+    storage_config,
+    retention_config,
+    local_path='borg',
+    remote_path=None,
+    stats=False,
 ):
     '''
     Given dry-run flag, a local or remote repository path, a storage config dict, and a
@@ -42,7 +51,7 @@ def prune_archives(
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'prune', repository)
+        (local_path, 'prune')
         + tuple(element for pair in _make_prune_flags(retention_config) for element in pair)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
@@ -51,7 +60,8 @@ def prune_archives(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
+        + (('--stats',) if stats else ())
+        + (repository,)
     )
 
-    logger.debug(' '.join(full_command))
-    subprocess.check_call(full_command)
+    execute_command(full_command, output_log_level=logging.WARNING if stats else logging.INFO)

borgmatic/commands/arguments.py

@@ -0,0 +1,461 @@
+import collections
+from argparse import ArgumentParser
+
+from borgmatic.config import collect
+
+SUBPARSER_ALIASES = {
+    'init': ['--init', '-I'],
+    'prune': ['--prune', '-p'],
+    'create': ['--create', '-C'],
+    'check': ['--check', '-k'],
+    'extract': ['--extract', '-x'],
+    'restore': ['--restore', '-r'],
+    'list': ['--list', '-l'],
+    'info': ['--info', '-i'],
+}
+
+
+def parse_subparser_arguments(unparsed_arguments, subparsers):
+    '''
+    Given a sequence of arguments, and a subparsers object as returned by
+    argparse.ArgumentParser().add_subparsers(), give each requested action's subparser a shot at
+    parsing all arguments. This allows common arguments like "--repository" to be shared across
+    multiple subparsers.
+
+    Return the result as a dict mapping from subparser name to a parsed namespace of arguments.
+    '''
+    arguments = collections.OrderedDict()
+    remaining_arguments = list(unparsed_arguments)
+    alias_to_subparser_name = {
+        alias: subparser_name
+        for subparser_name, aliases in SUBPARSER_ALIASES.items()
+        for alias in aliases
+    }
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        canonical_name = alias_to_subparser_name.get(subparser_name, subparser_name)
+
+        # If a parsed value happens to be the same as the name of a subparser, remove it from the
+        # remaining arguments. This prevents, for instance, "check --only extract" from triggering
+        # the "extract" subparser.
+        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+        for value in vars(parsed).values():
+            if isinstance(value, str):
+                if value in subparsers.choices:
+                    remaining_arguments.remove(value)
+            elif isinstance(value, list):
+                for item in value:
+                    if item in subparsers.choices:
+                        remaining_arguments.remove(item)
+
+        arguments[canonical_name] = parsed
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+            arguments[subparser_name] = parsed
+
+    return arguments
+
+
+def parse_global_arguments(unparsed_arguments, top_level_parser, subparsers):
+    '''
+    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
+    object as returned by argparse.ArgumentParser().add_subparsers(), parse and return any global
+    arguments as a parsed argparse.Namespace instance.
+    '''
+    # Ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
+    # are global arguments.
+    remaining_arguments = list(unparsed_arguments)
+    present_subparser_names = set()
+
+    for subparser_name, subparser in subparsers.choices.items():
+        if subparser_name not in remaining_arguments:
+            continue
+
+        present_subparser_names.add(subparser_name)
+        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # If no actions are explicitly requested, assume defaults: prune, create, and check.
+    if (
+        not present_subparser_names
+        and '--help' not in unparsed_arguments
+        and '-h' not in unparsed_arguments
+    ):
+        for subparser_name in ('prune', 'create', 'check'):
+            subparser = subparsers.choices[subparser_name]
+            unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+
+    # Remove the subparser names themselves.
+    for subparser_name in present_subparser_names:
+        if subparser_name in remaining_arguments:
+            remaining_arguments.remove(subparser_name)
+
+    return top_level_parser.parse_args(remaining_arguments)
+
+
+def parse_arguments(*unparsed_arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    global_parser = ArgumentParser(add_help=False)
+    global_group = global_parser.add_argument_group('global arguments')
+
+    global_group.add_argument(
+        '-c',
+        '--config',
+        nargs='*',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+    global_group.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    global_group.add_argument(
+        '-n',
+        '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write to any repositories',
+    )
+    global_group.add_argument(
+        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
+    )
+    global_group.add_argument(
+        '-v',
+        '--verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Display verbose progress to the console (from none to lots: 0, 1, or 2) or only errors (-1)',
+    )
+    global_group.add_argument(
+        '--syslog-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Log verbose progress to syslog (from none to lots: 0, 1, or 2) or only errors (-1). Ignored when console is interactive or --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Log verbose progress to log file (from none to lots: 0, 1, or 2) or only errors (-1). Only used when --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file',
+        type=str,
+        default=None,
+        help='Write log messages to this file instead of syslog',
+    )
+    global_group.add_argument(
+        '--version',
+        dest='version',
+        default=False,
+        action='store_true',
+        help='Display installed version number of borgmatic and exit',
+    )
+
+    top_level_parser = ArgumentParser(
+        description='''
+            A simple wrapper script for the Borg backup software that creates and prunes backups.
+            If none of the action options are given, then borgmatic defaults to: prune, create, and
+            check archives.
+            ''',
+        parents=[global_parser],
+    )
+
+    subparsers = top_level_parser.add_subparsers(
+        title='actions',
+        metavar='',
+        help='Specify zero or more actions. Defaults to prune, create, and check. Use --help with action for details:',
+    )
+    init_parser = subparsers.add_parser(
+        'init',
+        aliases=SUBPARSER_ALIASES['init'],
+        help='Initialize an empty Borg repository',
+        description='Initialize an empty Borg repository',
+        add_help=False,
+    )
+    init_group = init_parser.add_argument_group('init arguments')
+    init_group.add_argument(
+        '-e',
+        '--encryption',
+        dest='encryption_mode',
+        help='Borg repository encryption mode',
+        required=True,
+    )
+    init_group.add_argument(
+        '--append-only',
+        dest='append_only',
+        action='store_true',
+        help='Create an append-only repository',
+    )
+    init_group.add_argument(
+        '--storage-quota',
+        dest='storage_quota',
+        help='Create a repository with a fixed storage quota',
+    )
+    init_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    prune_parser = subparsers.add_parser(
+        'prune',
+        aliases=SUBPARSER_ALIASES['prune'],
+        help='Prune archives according to the retention policy',
+        description='Prune archives according to the retention policy',
+        add_help=False,
+    )
+    prune_group = prune_parser.add_argument_group('prune arguments')
+    prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    create_parser = subparsers.add_parser(
+        'create',
+        aliases=SUBPARSER_ALIASES['create'],
+        help='Create archives (actually perform backups)',
+        description='Create archives (actually perform backups)',
+        add_help=False,
+    )
+    create_group = create_parser.add_argument_group('create arguments')
+    create_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    create_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+    create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    check_parser = subparsers.add_parser(
+        'check',
+        aliases=SUBPARSER_ALIASES['check'],
+        help='Check archives for consistency',
+        description='Check archives for consistency',
+        add_help=False,
+    )
+    check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--only',
+        metavar='CHECK',
+        choices=('repository', 'archives', 'data', 'extract'),
+        dest='only',
+        action='append',
+        help='Run a particular consistency check (repository, archives, data, or extract) instead of configured checks; can specify flag multiple times',
+    )
+    check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    extract_parser = subparsers.add_parser(
+        'extract',
+        aliases=SUBPARSER_ALIASES['extract'],
+        help='Extract files from a named archive to the current directory',
+        description='Extract a named archive to the current directory',
+        add_help=False,
+    )
+    extract_group = extract_parser.add_argument_group('extract arguments')
+    extract_group.add_argument(
+        '--repository',
+        help='Path of repository to extract, defaults to the configured repository if there is only one',
+    )
+    extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
+    extract_group.add_argument(
+        '--path',
+        '--restore-path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to extract from archive, defaults to the entire archive',
+    )
+    extract_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Directory to extract files into, defaults to the current directory',
+    )
+    extract_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    extract_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    restore_parser = subparsers.add_parser(
+        'restore',
+        aliases=SUBPARSER_ALIASES['restore'],
+        help='Restore database dumps from a named archive',
+        description='Restore database dumps from a named archive. (To extract files instead, use "borgmatic extract".)',
+        add_help=False,
+    )
+    restore_group = restore_parser.add_argument_group('restore arguments')
+    restore_group.add_argument(
+        '--repository',
+        help='Path of repository to restore from, defaults to the configured repository if there is only one',
+    )
+    restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
+    restore_group.add_argument(
+        '--database',
+        metavar='NAME',
+        nargs='+',
+        dest='databases',
+        help='Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each database dump file as it is extracted from archive',
+    )
+    restore_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    list_parser = subparsers.add_parser(
+        'list',
+        aliases=SUBPARSER_ALIASES['list'],
+        help='List archives',
+        description='List archives or the contents of an archive',
+        add_help=False,
+    )
+    list_group = list_parser.add_argument_group('list arguments')
+    list_group.add_argument(
+        '--repository',
+        help='Path of repository to list, defaults to the configured repository if there is only one',
+    )
+    list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument(
+        '--short', default=False, action='store_true', help='Output only archive or path names'
+    )
+    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument(
+        '--json', default=False, action='store_true', help='Output results as JSON'
+    )
+    list_group.add_argument(
+        '-P', '--prefix', help='Only list archive names starting with this prefix'
+    )
+    list_group.add_argument(
+        '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
+    )
+    list_group.add_argument(
+        '--successful',
+        default=False,
+        action='store_true',
+        help='Only list archive names of successful (non-checkpoint) backups',
+    )
+    list_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    list_group.add_argument(
+        '--first', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '--last', metavar='N', help='List last N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+    )
+    list_group.add_argument(
+        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+    )
+    list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
+    list_group.add_argument(
+        '--patterns-from',
+        metavar='FILENAME',
+        help='Include or exclude paths matching patterns from pattern file, one per line',
+    )
+    list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    info_parser = subparsers.add_parser(
+        'info',
+        aliases=SUBPARSER_ALIASES['info'],
+        help='Display summary information on archives',
+        description='Display summary information on archives',
+        add_help=False,
+    )
+    info_group = info_parser.add_argument_group('info arguments')
+    info_group.add_argument(
+        '--repository',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one',
+    )
+    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+    info_group.add_argument(
+        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+    )
+    info_group.add_argument(
+        '-a',
+        '--glob-archives',
+        metavar='GLOB',
+        help='Only show info for archive names matching this glob',
+    )
+    info_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    info_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Show info for first N archives after other filters are applied',
+    )
+    info_group.add_argument(
+        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+    )
+    info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments = parse_subparser_arguments(unparsed_arguments, subparsers)
+    arguments['global'] = parse_global_arguments(unparsed_arguments, top_level_parser, subparsers)
+
+    if arguments['global'].excludes_filename:
+        raise ValueError(
+            'The --excludes option has been replaced with exclude_patterns in configuration'
+        )
+
+    if 'init' in arguments and arguments['global'].dry_run:
+        raise ValueError('The init action cannot be used with the --dry-run option')
+
+    if 'list' in arguments and arguments['list'].glob_archives and arguments['list'].successful:
+        raise ValueError('The --glob-archives and --successful options cannot be used together')
+
+    if (
+        'list' in arguments
+        and 'info' in arguments
+        and arguments['list'].json
+        and arguments['info'].json
+    ):
+        raise ValueError('With the --json option, list and info actions cannot be used together')
+
+    return arguments

borgmatic/commands/borgmatic.py

@@ -1,262 +1,549 @@
-from argparse import ArgumentParser
+import collections
 import json
 import logging
 import os
-from subprocess import CalledProcessError
 import sys
+from subprocess import CalledProcessError
 
-from borgmatic.borg import (
-    check as borg_check,
-    create as borg_create,
-    prune as borg_prune,
-    list as borg_list,
-    info as borg_info,
-)
-from borgmatic.commands import hook
+import colorama
+import pkg_resources
+
+from borgmatic.borg import check as borg_check
+from borgmatic.borg import create as borg_create
+from borgmatic.borg import environment as borg_environment
+from borgmatic.borg import extract as borg_extract
+from borgmatic.borg import info as borg_info
+from borgmatic.borg import init as borg_init
+from borgmatic.borg import list as borg_list
+from borgmatic.borg import prune as borg_prune
+from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
+from borgmatic.hooks import command, cronhub, cronitor, dispatch, dump, healthchecks
+from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
 
-
 logger = logging.getLogger(__name__)
 
-
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
 
 
-def parse_arguments(*arguments):
+def run_configuration(config_filename, config, arguments):
     '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
+    Given a config filename, the corresponding parsed config dict, and command-line arguments as a
+    dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
+    backups, consistency checks, and/or other actions.
+
+    Yield a combination of:
+
+      * JSON output strings from successfully executing any actions that produce JSON
+      * logging.LogRecord instances containing errors from any actions or backup hooks that fail
     '''
-    config_paths = collect.get_default_config_paths()
-
-    parser = ArgumentParser(
-        description='''
-            A simple wrapper script for the Borg backup software that creates and prunes backups.
-            If none of the --prune, --create, or --check options are given, then borgmatic defaults
-            to all three: prune, create, and check archives.
-            '''
-    )
-    parser.add_argument(
-        '-c',
-        '--config',
-        nargs='+',
-        dest='config_paths',
-        default=config_paths,
-        help='Configuration filenames or directories, defaults to: {}'.format(
-            ' '.join(config_paths)
-        ),
-    )
-    parser.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        help='Deprecated in favor of exclude_patterns within configuration',
-    )
-    parser.add_argument(
-        '-p',
-        '--prune',
-        dest='prune',
-        action='store_true',
-        help='Prune archives according to the retention policy',
-    )
-    parser.add_argument(
-        '-C',
-        '--create',
-        dest='create',
-        action='store_true',
-        help='Create archives (actually perform backups)',
-    )
-    parser.add_argument(
-        '-k', '--check', dest='check', action='store_true', help='Check archives for consistency'
-    )
-    parser.add_argument('-l', '--list', dest='list', action='store_true', help='List archives')
-    parser.add_argument(
-        '-i',
-        '--info',
-        dest='info',
-        action='store_true',
-        help='Display summary information on archives',
-    )
-    parser.add_argument(
-        '--json',
-        dest='json',
-        default=False,
-        action='store_true',
-        help='Output results from the --create, --list, or --info options as json',
-    )
-    parser.add_argument(
-        '-n',
-        '--dry-run',
-        dest='dry_run',
-        action='store_true',
-        help='Go through the motions, but do not actually write to any repositories',
-    )
-    parser.add_argument(
-        '-v',
-        '--verbosity',
-        type=int,
-        choices=range(0, 3),
-        default=0,
-        help='Display verbose progress (1 for some, 2 for lots)',
-    )
-
-    args = parser.parse_args(arguments)
-
-    if args.json and not (args.create or args.list or args.info):
-        raise ValueError(
-            'The --json option can only be used with the --create, --list, or --info options'
-        )
-
-    if args.json and args.list and args.info:
-        raise ValueError(
-            'With the --json option, options --list and --info cannot be used together'
-        )
-
-    # If any of the action flags are explicitly requested, leave them as-is. Otherwise, assume
-    # defaults: Mutate the given arguments to enable the default actions.
-    if args.prune or args.create or args.check or args.list or args.info:
-        return args
-
-    args.prune = True
-    args.create = True
-    args.check = True
-    return args
-
-
-def run_configuration(config_filename, args):  # pragma: no cover
-    '''
-    Parse a single configuration file, and execute its defined pruning, backups, and/or consistency
-    checks.
-    '''
-    logger.info('{}: Parsing configuration file'.format(config_filename))
-    config = validate.parse_configuration(config_filename, validate.schema_filename())
     (location, storage, retention, consistency, hooks) = (
         config.get(section_name, {})
         for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
     )
+    global_arguments = arguments['global']
 
-    try:
-        local_path = location.get('local_path', 'borg')
-        remote_path = location.get('remote_path')
-        borg_create.initialize_environment(storage)
+    local_path = location.get('local_path', 'borg')
+    remote_path = location.get('remote_path')
+    borg_environment.initialize(storage)
+    encountered_error = None
+    error_repository = ''
 
-        if args.create:
-            hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
+    if 'create' in arguments:
+        try:
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
+            )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
+            )
+            cronhub.ping_cronhub(
+                hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'start'
+            )
+            command.execute_hook(
+                hooks.get('before_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'pre-backup',
+                global_arguments.dry_run,
+            )
+            dispatch.call_hooks(
+                'dump_databases',
+                hooks,
+                config_filename,
+                dump.DATABASE_HOOK_NAMES,
+                global_arguments.dry_run,
+            )
+        except (OSError, CalledProcessError) as error:
+            encountered_error = error
+            yield from make_error_log_records(
+                '{}: Error running pre-backup hook'.format(config_filename), error
+            )
 
-        _run_commands(
-            args=args,
-            consistency=consistency,
-            local_path=local_path,
-            location=location,
-            remote_path=remote_path,
-            retention=retention,
-            storage=storage,
-        )
+    if not encountered_error:
+        for repository_path in location['repositories']:
+            try:
+                yield from run_actions(
+                    arguments=arguments,
+                    location=location,
+                    storage=storage,
+                    retention=retention,
+                    consistency=consistency,
+                    hooks=hooks,
+                    local_path=local_path,
+                    remote_path=remote_path,
+                    repository_path=repository_path,
+                )
+            except (OSError, CalledProcessError, ValueError) as error:
+                encountered_error = error
+                error_repository = repository_path
+                yield from make_error_log_records(
+                    '{}: Error running actions for repository'.format(repository_path), error
+                )
 
-        if args.create:
-            hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
-    except (OSError, CalledProcessError):
-        hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
-        raise
+    if 'create' in arguments and not encountered_error:
+        try:
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                config_filename,
+                dump.DATABASE_HOOK_NAMES,
+                global_arguments.dry_run,
+            )
+            command.execute_hook(
+                hooks.get('after_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'post-backup',
+                global_arguments.dry_run,
+            )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run
+            )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
+            )
+            cronhub.ping_cronhub(
+                hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'finish'
+            )
+        except (OSError, CalledProcessError) as error:
+            encountered_error = error
+            yield from make_error_log_records(
+                '{}: Error running post-backup hook'.format(config_filename), error
+            )
+
+    if encountered_error:
+        try:
+            command.execute_hook(
+                hooks.get('on_error'),
+                hooks.get('umask'),
+                config_filename,
+                'on-error',
+                global_arguments.dry_run,
+                repository=error_repository,
+                error=encountered_error,
+                output=getattr(encountered_error, 'output', ''),
+            )
+            healthchecks.ping_healthchecks(
+                hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
+            )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
+            )
+            cronhub.ping_cronhub(
+                hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'fail'
+            )
+        except (OSError, CalledProcessError) as error:
+            yield from make_error_log_records(
+                '{}: Error running on-error hook'.format(config_filename), error
+            )
 
 
-def _run_commands(*, args, consistency, local_path, location, remote_path, retention, storage):
-    json_results = []
-    for unexpanded_repository in location['repositories']:
-        _run_commands_on_repository(
-            args=args,
-            consistency=consistency,
-            json_results=json_results,
-            local_path=local_path,
-            location=location,
-            remote_path=remote_path,
-            retention=retention,
-            storage=storage,
-            unexpanded_repository=unexpanded_repository,
-        )
-    if args.json:
-        sys.stdout.write(json.dumps(json_results))
-
-
-def _run_commands_on_repository(
+def run_actions(
     *,
-    args,
-    consistency,
-    json_results,
-    local_path,
+    arguments,
     location,
-    remote_path,
-    retention,
     storage,
-    unexpanded_repository,
+    retention,
+    consistency,
+    hooks,
+    local_path,
+    remote_path,
+    repository_path
 ):  # pragma: no cover
-    repository = os.path.expanduser(unexpanded_repository)
-    dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
-    if args.prune:
+    '''
+    Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
+    configuration dicts, local and remote paths to Borg, and a repository name, run all actions
+    from the command-line arguments on the given repository.
+
+    Yield JSON output strings from executing any actions that produce JSON.
+
+    Raise OSError or subprocess.CalledProcessError if an error occurs running a command for an
+    action. Raise ValueError if the arguments or configuration passed to action are invalid.
+    '''
+    repository = os.path.expanduser(repository_path)
+    global_arguments = arguments['global']
+    dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
+    if 'init' in arguments:
+        logger.info('{}: Initializing repository'.format(repository))
+        borg_init.initialize_repository(
+            repository,
+            arguments['init'].encryption_mode,
+            arguments['init'].append_only,
+            arguments['init'].storage_quota,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+    if 'prune' in arguments:
         logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
         borg_prune.prune_archives(
-            args.dry_run,
+            global_arguments.dry_run,
             repository,
             storage,
             retention,
             local_path=local_path,
             remote_path=remote_path,
+            stats=arguments['prune'].stats,
         )
-    if args.create:
+    if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
-        borg_create.create_archive(
-            args.dry_run,
+        json_output = borg_create.create_archive(
+            global_arguments.dry_run,
             repository,
             location,
             storage,
             local_path=local_path,
             remote_path=remote_path,
+            progress=arguments['create'].progress,
+            stats=arguments['create'].stats,
+            json=arguments['create'].json,
         )
-    if args.check and checks.repository_enabled_for_checks(repository, consistency):
+        if json_output:
+            yield json.loads(json_output)
+    if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
-            repository, storage, consistency, local_path=local_path, remote_path=remote_path
+            repository,
+            storage,
+            consistency,
+            local_path=local_path,
+            remote_path=remote_path,
+            only_checks=arguments['check'].only,
         )
-    if args.list:
-        logger.info('{}: Listing archives'.format(repository))
-        output = borg_list.list_archives(
-            repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
+    if 'extract' in arguments:
+        if arguments['extract'].repository is None or repository == arguments['extract'].repository:
+            logger.info(
+                '{}: Extracting archive {}'.format(repository, arguments['extract'].archive)
+            )
+            borg_extract.extract_archive(
+                global_arguments.dry_run,
+                repository,
+                arguments['extract'].archive,
+                arguments['extract'].paths,
+                location,
+                storage,
+                local_path=local_path,
+                remote_path=remote_path,
+                destination_path=arguments['extract'].destination,
+                progress=arguments['extract'].progress,
+            )
+    if 'restore' in arguments:
+        if arguments['restore'].repository is None or repository == arguments['restore'].repository:
+            logger.info(
+                '{}: Restoring databases from archive {}'.format(
+                    repository, arguments['restore'].archive
+                )
+            )
+
+            restore_names = arguments['restore'].databases or []
+            if 'all' in restore_names:
+                restore_names = []
+
+            # Extract dumps for the named databases from the archive.
+            dump_patterns = dispatch.call_hooks(
+                'make_database_dump_patterns',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                restore_names,
+            )
+            borg_extract.extract_archive(
+                global_arguments.dry_run,
+                repository,
+                arguments['restore'].archive,
+                dump.convert_glob_patterns_to_borg_patterns(
+                    [pattern for patterns in dump_patterns.values() for pattern in patterns]
+                ),
+                location,
+                storage,
+                local_path=local_path,
+                remote_path=remote_path,
+                destination_path='/',
+                progress=arguments['restore'].progress,
+                # We don't want glob patterns that don't match to error.
+                error_on_warnings=False,
+            )
+
+            # Map the restore names or detected dumps to the corresponding database configurations.
+            restore_databases = dump.get_per_hook_database_configurations(
+                hooks, restore_names, dump_patterns
+            )
+
+            # Finally, restore the databases and cleanup the dumps.
+            dispatch.call_hooks(
+                'restore_database_dumps',
+                restore_databases,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                global_arguments.dry_run,
+            )
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                restore_databases,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                global_arguments.dry_run,
+            )
+    if 'list' in arguments:
+        if arguments['list'].repository is None or repository == arguments['list'].repository:
+            logger.info('{}: Listing archives'.format(repository))
+            json_output = borg_list.list_archives(
+                repository,
+                storage,
+                list_arguments=arguments['list'],
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            if json_output:
+                yield json.loads(json_output)
+    if 'info' in arguments:
+        if arguments['info'].repository is None or repository == arguments['info'].repository:
+            logger.info('{}: Displaying summary info for archives'.format(repository))
+            json_output = borg_info.display_archives_info(
+                repository,
+                storage,
+                info_arguments=arguments['info'],
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            if json_output:
+                yield json.loads(json_output)
+
+
+def load_configurations(config_filenames):
+    '''
+    Given a sequence of configuration filenames, load and validate each configuration file. Return
+    the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
+    and sequence of logging.LogRecord instances containing any parse errors.
+    '''
+    # Dict mapping from config filename to corresponding parsed config dict.
+    configs = collections.OrderedDict()
+    logs = []
+
+    # Parse and load each configuration file.
+    for config_filename in config_filenames:
+        try:
+            configs[config_filename] = validate.parse_configuration(
+                config_filename, validate.schema_filename()
+            )
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logs.extend(
+                [
+                    logging.makeLogRecord(
+                        dict(
+                            levelno=logging.CRITICAL,
+                            levelname='CRITICAL',
+                            msg='{}: Error parsing configuration file'.format(config_filename),
+                        )
+                    ),
+                    logging.makeLogRecord(
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+                    ),
+                ]
+            )
+
+    return (configs, logs)
+
+
+def make_error_log_records(message, error=None):
+    '''
+    Given error message text and an optional exception object, yield a series of logging.LogRecord
+    instances with error summary information.
+    '''
+    if not error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
         )
-        if args.json:
-            json_results.append(json.loads(output))
+        return
+
+    try:
+        raise error
+    except CalledProcessError as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        if error.output:
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
+            )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except (ValueError, OSError) as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except:  # noqa: E722
+        # Raising above only as a means of determining the error type. Swallow the exception here
+        # because we don't want the exception to propagate out of this function.
+        pass
+
+
+def collect_configuration_run_summary_logs(configs, arguments):
+    '''
+    Given a dict of configuration filename to corresponding parsed configuration, and parsed
+    command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
+    each configuration file and yield a series of logging.LogRecord instances containing summary
+    information about each run.
+
+    As a side effect of running through these configuration files, output their JSON results, if
+    any, to stdout.
+    '''
+    # Run cross-file validation checks.
+    if 'extract' in arguments:
+        repository = arguments['extract'].repository
+    elif 'list' in arguments and arguments['list'].archive:
+        repository = arguments['list'].repository
+    else:
+        repository = None
+
+    if repository:
+        try:
+            validate.guard_configuration_contains_repository(repository, configs)
+        except ValueError as error:
+            yield from make_error_log_records(str(error))
+            return
+
+    if not configs:
+        yield from make_error_log_records(
+            '{}: No configuration files found'.format(' '.join(arguments['global'].config_paths))
+        )
+        return
+
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                command.execute_hook(
+                    hooks.get('before_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'pre-everything',
+                    arguments['global'].dry_run,
+                )
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running pre-everything hook', error)
+            return
+
+    # Execute the actions corresponding to each configuration file.
+    json_results = []
+    for config_filename, config in configs.items():
+        results = list(run_configuration(config_filename, config, arguments))
+        error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
+
+        if error_logs:
+            yield from make_error_log_records(
+                '{}: Error running configuration file'.format(config_filename)
+            )
+            yield from error_logs
         else:
-            sys.stdout.write(output)
-    if args.info:
-        logger.info('{}: Displaying summary info for archives'.format(repository))
-        output = borg_info.display_archives_info(
-            repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
-        )
-        if args.json:
-            json_results.append(json.loads(output))
-        else:
-            sys.stdout.write(output)
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.INFO,
+                    levelname='INFO',
+                    msg='{}: Successfully ran configuration file'.format(config_filename),
+                )
+            )
+            if results:
+                json_results.extend(results)
+
+    if json_results:
+        sys.stdout.write(json.dumps(json_results))
+
+    if 'create' in arguments:
+        try:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                command.execute_hook(
+                    hooks.get('after_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-everything',
+                    arguments['global'].dry_run,
+                )
+        except (CalledProcessError, ValueError, OSError) as error:
+            yield from make_error_log_records('Error running post-everything hook', error)
+
+
+def exit_with_help_link():  # pragma: no cover
+    '''
+    Display a link to get help and exit with an error code.
+    '''
+    logger.critical('')
+    logger.critical('Need some help? https://torsion.org/borgmatic/#issues')
+    sys.exit(1)
 
 
 def main():  # pragma: no cover
+    configure_signals()
+
     try:
-        configure_signals()
-        args = parse_arguments(*sys.argv[1:])
-        logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
+        arguments = parse_arguments(*sys.argv[1:])
+    except ValueError as error:
+        configure_logging(logging.CRITICAL)
+        logger.critical(error)
+        exit_with_help_link()
+    except SystemExit as error:
+        if error.code == 0:
+            raise error
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv)))
+        exit_with_help_link()
 
-        config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
-        logger.debug('Ensuring legacy configuration is upgraded')
-        convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
+    global_arguments = arguments['global']
+    if global_arguments.version:
+        print(pkg_resources.require('borgmatic')[0].version)
+        sys.exit(0)
 
-        if len(config_filenames) == 0:
-            raise ValueError(
-                'Error: No configuration files found in: {}'.format(' '.join(args.config_paths))
-            )
+    config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
+    configs, parse_logs = load_configurations(config_filenames)
 
-        for config_filename in config_filenames:
-            run_configuration(config_filename, args)
-    except (ValueError, OSError, CalledProcessError) as error:
-        print(error, file=sys.stderr)
-        print(file=sys.stderr)
-        print('Need some help? https://torsion.org/borgmatic/#issues', file=sys.stderr)
-        sys.exit(1)
+    colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
+    try:
+        configure_logging(
+            verbosity_to_log_level(global_arguments.verbosity),
+            verbosity_to_log_level(global_arguments.syslog_verbosity),
+            verbosity_to_log_level(global_arguments.log_file_verbosity),
+            global_arguments.log_file,
+        )
+    except (FileNotFoundError, PermissionError) as error:
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error configuring logging: {}'.format(error))
+        exit_with_help_link()
+
+    logger.debug('Ensuring legacy configuration is upgraded')
+    convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
+
+    summary_logs = list(collect_configuration_run_summary_logs(configs, arguments))
+
+    logger.info('')
+    logger.info('summary:')
+    [
+        logger.handle(log)
+        for log in parse_logs + summary_logs
+        if log.levelno >= logger.getEffectiveLevel()
+    ]
+
+    if any(log.levelno == logging.CRITICAL for log in summary_logs):
+        exit_with_help_link()

borgmatic/commands/convert_config.py

@@ -1,13 +1,12 @@
-from argparse import ArgumentParser
 import os
 import sys
 import textwrap
+from argparse import ArgumentParser
 
 from ruamel import yaml
 
 from borgmatic.config import convert, generate, legacy, validate
 
-
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'

borgmatic/commands/generate_config.py

@@ -1,9 +1,8 @@
-from argparse import ArgumentParser
 import sys
+from argparse import ArgumentParser
 
 from borgmatic.config import generate, validate
 
-
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
 
@@ -13,12 +12,18 @@ def parse_arguments(*arguments):
     them as an ArgumentParser instance.
     '''
     parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
+    parser.add_argument(
+        '-s',
+        '--source',
+        dest='source_filename',
+        help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration',
+    )
     parser.add_argument(
         '-d',
         '--destination',
         dest='destination_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(
+        help='Destination YAML configuration file. Default: {}'.format(
             DEFAULT_DESTINATION_CONFIG_FILENAME
         ),
     )
@@ -31,12 +36,22 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
 
         generate.generate_sample_configuration(
-            args.destination_filename, validate.schema_filename()
+            args.source_filename, args.destination_filename, validate.schema_filename()
         )
 
         print('Generated a sample configuration file at {}.'.format(args.destination_filename))
         print()
-        print('Please edit the file to suit your needs. The values are just representative.')
+        if args.source_filename:
+            print(
+                'Merged in the contents of configuration file at {}.'.format(args.source_filename)
+            )
+            print('To review the changes made, run:')
+            print()
+            print(
+                '    diff --unified {} {}'.format(args.source_filename, args.destination_filename)
+            )
+            print()
+        print('Please edit the file to suit your needs. The values are representative.')
         print('All fields are optional except where indicated.')
         print()
         print('If you ever need help: https://torsion.org/borgmatic/#issues')

borgmatic/commands/hook.py

@@ -1,24 +0,0 @@
-import logging
-import subprocess
-
-
-logger = logging.getLogger(__name__)
-
-
-def execute_hook(commands, config_filename, description):
-    if not commands:
-        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
-        return
-
-    if len(commands) == 1:
-        logger.info('{}: Running command for {} hook'.format(config_filename, description))
-    else:
-        logger.info(
-            '{}: Running {} commands for {} hook'.format(
-                config_filename, len(commands), description
-            )
-        )
-
-    for command in commands:
-        logger.debug('{}: Hook command: {}'.format(config_filename, command))
-        subprocess.check_call(command, shell=True)

borgmatic/commands/validate_config.py

@@ -0,0 +1,56 @@
+import logging
+import sys
+from argparse import ArgumentParser
+
+from borgmatic.config import collect, validate
+
+logger = logging.getLogger(__name__)
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
+    parser.add_argument(
+        '-c',
+        '--config',
+        nargs='+',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():  # pragma: no cover
+    args = parse_arguments(*sys.argv[1:])
+
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    if len(config_filenames) == 0:
+        logger.critical('No files to validate found')
+        sys.exit(1)
+
+    found_issues = False
+    for config_filename in config_filenames:
+        try:
+            validate.parse_configuration(config_filename, validate.schema_filename())
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logging.critical('{}: Error parsing configuration file'.format(config_filename))
+            logging.critical(error)
+            found_issues = True
+
+    if found_issues:
+        sys.exit(1)
+    else:
+        logger.info(
+            'All given configuration files are valid: {}'.format(', '.join(config_filenames))
+        )

borgmatic/config/collect.py

@@ -20,10 +20,10 @@ def get_default_config_paths():
 
 def collect_config_filenames(config_paths):
     '''
-    Given a sequence of config paths, both filenames and directories, resolve that to just an
-    iterable of files. Accomplish this by listing any given directories looking for contained config
-    files (ending with the ".yaml" extension). This is non-recursive, so any directories within the
-    given directories are ignored.
+    Given a sequence of config paths, both filenames and directories, resolve that to an iterable
+    of files. Accomplish this by listing any given directories looking for contained config files
+    (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given
+    directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
     configuration paths. However, skip a default config path if it's missing, so the user doesn't
@@ -41,7 +41,8 @@ def collect_config_filenames(config_paths):
             yield path
             continue
 
-        for filename in os.listdir(path):
+        for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
-            if full_filename.endswith('.yaml') and not os.path.isdir(full_filename):
+            matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
+            if matching_filetype and not os.path.isdir(full_filename):
                 yield full_filename

borgmatic/config/convert.py

@@ -54,19 +54,19 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
     # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration(destination_config, schema)
+    generate.add_comments_to_configuration_map(destination_config, schema)
 
     for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration(
+        generate.add_comments_to_configuration_map(
             section_config, schema['map'][section_name], indent=generate.INDENT
         )
 
     return destination_config
 
 
-class LegacyConfigurationNotUpgraded(FileNotFoundError):
+class Legacy_configuration_not_upgraded(FileNotFoundError):
     def __init__(self):
-        super(LegacyConfigurationNotUpgraded, self).__init__(
+        super(Legacy_configuration_not_upgraded, self).__init__(
             '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
 to YAML. This better supports validation, and has a more natural way to express
 lists of values. To upgrade your existing configuration, run:
@@ -83,7 +83,7 @@ instead of the old one.'''
 def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
     '''
     If legacy source configuration exists but no destination upgraded configs do, raise
-    LegacyConfigurationNotUpgraded.
+    Legacy_configuration_not_upgraded.
 
     The idea is that we want to alert the user about upgrading their config if they haven't already.
     '''
@@ -92,22 +92,4 @@ def guard_configuration_upgraded(source_config_filename, destination_config_file
     )
 
     if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise LegacyConfigurationNotUpgraded()
-
-
-class LegacyExcludesFilenamePresent(FileNotFoundError):
-    def __init__(self):
-        super(LegacyExcludesFilenamePresent, self).__init__(
-            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
-to YAML. This better supports validation, and has a more natural way to express
-lists of values. The new configuration file incorporates excludes, so you no
-longer need to provide an excludes filename on the command-line with an
-"--excludes" argument.
-
-Please remove the "--excludes" argument and run borgmatic again.'''
-        )
-
-
-def guard_excludes_filename_omitted(excludes_filename):
-    if excludes_filename is not None:
-        raise LegacyExcludesFilenamePresent()
+        raise Legacy_configuration_not_upgraded()

borgmatic/config/generate.py

@@ -1,9 +1,14 @@
+import collections
+import io
 import os
+import re
 
 from ruamel import yaml
 
+from borgmatic.config import load
 
 INDENT = 4
+SEQUENCE_INDENT = 2
 
 
 def _insert_newline_before_comment(config, field_name):
@@ -16,7 +21,7 @@ def _insert_newline_before_comment(config, field_name):
     )
 
 
-def _schema_to_sample_configuration(schema, level=0):
+def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema, generate and return sample config for it. Include comments
     for each section based on the schema "desc" description.
@@ -25,14 +30,29 @@ def _schema_to_sample_configuration(schema, level=0):
     if example is not None:
         return example
 
-    config = yaml.comments.CommentedMap(
-        [
-            (section_name, _schema_to_sample_configuration(section_schema, level + 1))
-            for section_name, section_schema in schema['map'].items()
-        ]
-    )
-
-    add_comments_to_configuration(config, schema, indent=(level * INDENT))
+    if 'seq' in schema:
+        config = yaml.comments.CommentedSeq(
+            [
+                _schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
+                for item_schema in schema['seq']
+            ]
+        )
+        add_comments_to_configuration_sequence(
+            config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
+        )
+    elif 'map' in schema:
+        config = yaml.comments.CommentedMap(
+            [
+                (field_name, _schema_to_sample_configuration(sub_schema, level + 1))
+                for field_name, sub_schema in schema['map'].items()
+            ]
+        )
+        indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
+        add_comments_to_configuration_map(
+            config, schema, indent=indent, skip_first=parent_is_sequence
+        )
+    else:
+        raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
 
     return config
 
@@ -43,46 +63,40 @@ def _comment_out_line(line):
     if not stripped_line or stripped_line.startswith('#'):
         return line
 
-    # Comment out the names of optional sections.
-    one_indent = ' ' * INDENT
-    if not line.startswith(one_indent):
-        return '#' + line
+    # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
+    matches = re.match(r'(\s*)', line)
+    indent_spaces = matches.group(0) if matches else ''
+    count_indent_spaces = len(indent_spaces)
 
-    # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
-    return '#'.join((one_indent, line[INDENT:]))
-
-
-REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
-REQUIRED_SECTION_NAMES = {'location', 'retention'}
+    return '# '.join((indent_spaces, line[count_indent_spaces:]))
 
 
 def _comment_out_optional_configuration(rendered_config):
     '''
-    Post-process a rendered configuration string to comment out optional key/values. The idea is
-    that this prevents the user from having to comment out a bunch of configuration they don't care
-    about to get to a minimal viable configuration file.
+    Post-process a rendered configuration string to comment out optional key/values, as determined
+    by a sentinel in the comment before each key.
 
-    Ideally ruamel.yaml would support this during configuration generation, but it's not terribly
-    easy to accomplish that way.
+    The idea is that the pre-commented configuration prevents the user from having to comment out a
+    bunch of configuration they don't care about to get to a minimal viable configuration file.
+
+    Ideally ruamel.yaml would support commenting out keys during configuration generation, but it's
+    not terribly easy to accomplish that way.
     '''
     lines = []
-    required = False
+    optional = False
 
     for line in rendered_config.split('\n'):
-        key = line.strip().split(':')[0]
-
-        if key in REQUIRED_SECTION_NAMES:
-            lines.append(line)
+        # Upon encountering an optional configuration option, commenting out lines until the next
+        # blank line.
+        if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)):
+            optional = True
             continue
 
-        # Upon encountering a required configuration option, skip commenting out lines until the
-        # next blank line.
-        if key in REQUIRED_KEYS:
-            required = True
-        elif not key:
-            required = False
+        # Hit a blank line, so reset commenting.
+        if not line.strip():
+            optional = False
 
-        lines.append(_comment_out_line(line) if not required else line)
+        lines.append(_comment_out_line(line) if optional else line)
 
     return '\n'.join(lines)
 
@@ -91,7 +105,12 @@ def _render_configuration(config):
     '''
     Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
     '''
-    return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)
+    dumper = yaml.YAML()
+    dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
+    rendered = io.StringIO()
+    dumper.dump(config, rendered)
+
+    return rendered.getvalue()
 
 
 def write_configuration(config_filename, rendered_config, mode=0o600):
@@ -113,33 +132,159 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
     os.chmod(config_filename, mode)
 
 
-def add_comments_to_configuration(config, schema, indent=0):
+def add_comments_to_configuration_sequence(config, schema, indent=0):
     '''
-    Using descriptions from a schema as a source, add those descriptions as comments to the given
-    config before each field. This function only adds comments for the top-most config map level.
-    Indent the comment the given number of characters.
+    If the given config sequence's items are maps, then mine the schema for the description of the
+    map's first item, and slap that atop the sequence. Indent the comment the given number of
+    characters.
+
+    Doing this for sequences of maps results in nice comments that look like:
+
+    ```
+    things:
+          # First key description. Added by this function.
+        - key: foo
+          # Second key description. Added by add_comments_to_configuration_map().
+          other: bar
+    ```
     '''
-    for index, field_name in enumerate(config.keys()):
-        field_schema = schema['map'].get(field_name, {})
+    if 'map' not in schema['seq'][0]:
+        return
+
+    for field_name in config[0].keys():
+        field_schema = schema['seq'][0]['map'].get(field_name, {})
         description = field_schema.get('desc')
 
         # No description to use? Skip it.
         if not field_schema or not description:
+            return
+
+        config[0].yaml_set_start_comment(description, indent=indent)
+
+        # We only want the first key's description here, as the rest of the keys get commented by
+        # add_comments_to_configuration_map().
+        return
+
+
+REQUIRED_SECTION_NAMES = {'location', 'retention'}
+REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
+COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
+
+
+def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
+    '''
+    Using descriptions from a schema as a source, add those descriptions as comments to the given
+    config mapping, before each field. Indent the comment the given number of characters.
+    '''
+    for index, field_name in enumerate(config.keys()):
+        if skip_first and index == 0:
+            continue
+
+        field_schema = schema['map'].get(field_name, {})
+        description = field_schema.get('desc', '').strip()
+
+        # If this is an optional key, add an indicator to the comment flagging it to be commented
+        # out from the sample configuration. This sentinel is consumed by downstream processing that
+        # does the actual commenting out.
+        if field_name not in REQUIRED_SECTION_NAMES and field_name not in REQUIRED_KEYS:
+            description = (
+                '\n'.join((description, COMMENTED_OUT_SENTINEL))
+                if description
+                else COMMENTED_OUT_SENTINEL
+            )
+
+        # No description to use? Skip it.
+        if not field_schema or not description:  # pragma: no cover
             continue
 
         config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
+
         if index > 0:
             _insert_newline_before_comment(config, field_name)
 
 
-def generate_sample_configuration(config_filename, schema_filename):
+RUAMEL_YAML_COMMENTS_INDEX = 1
+
+
+def remove_commented_out_sentinel(config, field_name):
     '''
-    Given a target config filename and the path to a schema filename in pykwalify YAML schema
-    format, write out a sample configuration file based on that schema.
+    Given a configuration CommentedMap and a top-level field name in it, remove any "commented out"
+    sentinel found at the end of its YAML comments. This prevents the given field name from getting
+    commented out by downstream processing that consumes the sentinel.
+    '''
+    try:
+        last_comment_value = config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX][-1].value
+    except KeyError:
+        return
+
+    if last_comment_value == '# {}\n'.format(COMMENTED_OUT_SENTINEL):
+        config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop()
+
+
+def merge_source_configuration_into_destination(destination_config, source_config):
+    '''
+    Deep merge the given source configuration dict into the destination configuration CommentedMap,
+    favoring values from the source when there are collisions.
+
+    The purpose of this is to upgrade configuration files from old versions of borgmatic by adding
+    new
+    configuration keys and comments.
+    '''
+    if not source_config:
+        return destination_config
+    if not destination_config or not isinstance(source_config, collections.abc.Mapping):
+        return source_config
+
+    for field_name, source_value in source_config.items():
+        # Since this key/value is from the source configuration, leave it uncommented and remove any
+        # sentinel that would cause it to get commented out.
+        remove_commented_out_sentinel(destination_config, field_name)
+
+        # This is a mapping. Recurse for this key/value.
+        if isinstance(source_value, collections.abc.Mapping):
+            destination_config[field_name] = merge_source_configuration_into_destination(
+                destination_config[field_name], source_value
+            )
+            continue
+
+        # This is a sequence. Recurse for each item in it.
+        if isinstance(source_value, collections.abc.Sequence) and not isinstance(source_value, str):
+            destination_value = destination_config[field_name]
+            destination_config[field_name] = yaml.comments.CommentedSeq(
+                [
+                    merge_source_configuration_into_destination(
+                        destination_value[index] if index < len(destination_value) else None,
+                        source_item,
+                    )
+                    for index, source_item in enumerate(source_value)
+                ]
+            )
+            continue
+
+        # This is some sort of scalar. Simply set it into the destination.
+        destination_config[field_name] = source_config[field_name]
+
+    return destination_config
+
+
+def generate_sample_configuration(source_filename, destination_filename, schema_filename):
+    '''
+    Given an optional source configuration filename, and a required destination configuration
+    filename, and the path to a schema filename in pykwalify YAML schema format, write out a
+    sample configuration file based on that schema. If a source filename is provided, merge the
+    parsed contents of that configuration into the generated configuration.
     '''
     schema = yaml.round_trip_load(open(schema_filename))
-    config = _schema_to_sample_configuration(schema)
+    source_config = None
+
+    if source_filename:
+        source_config = load.load_configuration(source_filename)
+
+    destination_config = merge_source_configuration_into_destination(
+        _schema_to_sample_configuration(schema), source_config
+    )
 
     write_configuration(
-        config_filename, _comment_out_optional_configuration(_render_configuration(config))
+        destination_filename,
+        _comment_out_optional_configuration(_render_configuration(destination_config)),
     )

borgmatic/config/legacy.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict, namedtuple
 from configparser import RawConfigParser
 
-
 Section_format = namedtuple('Section_format', ('name', 'options'))
 Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
 

borgmatic/config/load.py

@@ -0,0 +1,59 @@
+import logging
+import os
+
+import ruamel.yaml
+
+logger = logging.getLogger(__name__)
+
+
+def load_configuration(filename):
+    '''
+    Load the given configuration file and return its contents as a data structure of nested dicts
+    and lists.
+
+    Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
+    if there are too many recursive includes.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.Constructor = Include_constructor
+
+    return yaml.load(open(filename))
+
+
+def include_configuration(loader, filename_node):
+    '''
+    Load the given YAML filename (ignoring the given loader so we can use our own), and return its
+    contents as a data structure of nested dicts and lists.
+    '''
+    return load_configuration(os.path.expanduser(filename_node.value))
+
+
+class Include_constructor(ruamel.yaml.SafeConstructor):
+    '''
+    A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
+    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    '''
+
+    def __init__(self, preserve_quotes=None, loader=None):
+        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        self.add_constructor('!include', include_configuration)
+
+    def flatten_mapping(self, node):
+        '''
+        Support the special case of shallow merging included configuration into an existing mapping
+        using the YAML '<<' merge key. Example syntax:
+
+        ```
+        retention:
+            keep_daily: 1
+            <<: !include common.yaml
+        ```
+        '''
+        representer = ruamel.yaml.representer.SafeRepresenter()
+
+        for index, (key_node, value_node) in enumerate(node.value):
+            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+                included_value = representer.represent_data(self.construct_object(value_node))
+                node.value[index] = (key_node, included_value)
+
+        super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -11,7 +11,7 @@ map:
             source_directories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     List of source directories to backup (required). Globs and tildes are expanded.
                 example:
@@ -21,7 +21,7 @@ map:
             repositories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Paths to local or remote repositories (required). Tildes are expanded. Multiple
                     repositories are backed up to in sequence. See ssh_command for SSH options like
@@ -30,37 +30,53 @@ map:
                     - user@backupserver:sourcehostname.borg
             one_file_system:
                 type: bool
-                desc: Stay in same file system (do not cross mount points).
+                desc: Stay in same file system (do not cross mount points). Defaults to false.
                 example: true
+            numeric_owner:
+                type: bool
+                desc: Only store/extract numeric user and group identifiers. Defaults to false.
+                example: true
+            atime:
+                type: bool
+                desc: Store atime into archive. Defaults to true.
+                example: false
+            ctime:
+                type: bool
+                desc: Store ctime into archive. Defaults to true.
+                example: false
+            birthtime:
+                type: bool
+                desc: Store birthtime (creation date) into archive. Defaults to true.
+                example: false
             read_special:
                 type: bool
                 desc: |
                     Use Borg's --read-special flag to allow backup of block and other special
                     devices. Use with caution, as it will lead to problems if used when
-                    backing up special devices such as /dev/zero.
+                    backing up special devices such as /dev/zero. Defaults to false.
                 example: false
             bsd_flags:
                 type: bool
                 desc: Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true.
                 example: true
             files_cache:
-                type: scalar
+                type: str
                 desc: |
                     Mode in which to operate the files cache. See
                     https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
-                    details.
+                    details. Defaults to "ctime,size,inode".
                 example: ctime,size,inode
             local_path:
-                type: scalar
+                type: str
                 desc: Alternate Borg local executable. Defaults to "borg".
                 example: borg1
             remote_path:
-                type: scalar
+                type: str
                 desc: Alternate Borg remote executable. Defaults to "borg".
                 example: borg1
             patterns:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Any paths matching these patterns are included/excluded from backups. Globs are
                     expanded. (Tildes are not.) Note that Borg considers this option experimental.
@@ -73,7 +89,7 @@ map:
                     - '- /home/*'
             patterns_from:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Read include/exclude patterns from one or more separate named files, one pattern
                     per line. Note that Borg considers this option experimental. See the output of
@@ -82,7 +98,7 @@ map:
                     - /etc/borgmatic/patterns
             exclude_patterns:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Any paths matching these patterns are excluded from backups. Globs and tildes
                     are expanded. See the output of "borg help patterns" for more details.
@@ -92,7 +108,7 @@ map:
                     - /etc/ssl
             exclude_from:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Read exclude patterns from one or more separate named files, one pattern per
                     line. See the output of "borg help patterns" for more details.
@@ -102,12 +118,25 @@ map:
                 type: bool
                 desc: |
                     Exclude directories that contain a CACHEDIR.TAG file. See
-                    http://www.brynosaurus.com/cachedir/spec.html for details.
+                    http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false.
                 example: true
             exclude_if_present:
-                type: scalar
-                desc: Exclude directories that contain a file with the given filename.
+                type: str
+                desc: |
+                    Exclude directories that contain a file with the given filename. Defaults to not
+                    set.
                 example: .nobackup
+            keep_exclude_tags:
+                type: bool
+                desc: |
+                    If true, the exclude_if_present filename is included in backups. Defaults to
+                    false, meaning that the exclude_if_present filename is omitted from backups.
+                example: true
+            exclude_nodump:
+                type: bool
+                desc: |
+                    Exclude files with the NODUMP flag. Defaults to false.
+                example: true
     storage:
         desc: |
             Repository storage options. See
@@ -116,20 +145,20 @@ map:
             details.
         map:
             encryption_passcommand:
-                type: scalar
+                type: str
                 desc: |
                     The standard output of this command is used to unlock the encryption key. Only
                     use on repositories that were initialized with passcommand/repokey encryption.
                     Note that if both encryption_passcommand and encryption_passphrase are set,
-                    then encryption_passphrase takes precedence.
+                    then encryption_passphrase takes precedence. Defaults to not set.
                 example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
-                type: scalar
+                type: str
                 desc: |
                     Passphrase to unlock the encryption key with. Only use on repositories that were
                     initialized with passphrase/repokey encryption. Quote the value if it contains
                     punctuation, so it parses correctly. And backslash any quote or backslash
-                    literals as well.
+                    literals as well. Defaults to not set.
                 example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
             checkpoint_interval:
                 type: int
@@ -138,47 +167,94 @@ map:
                     https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there
                     for details. Defaults to checkpoints every 1800 seconds (30 minutes).
                 example: 1800
+            chunker_params:
+                type: str
+                desc: |
+                    Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
+                    HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
+                    for details. Defaults to "19,23,21,4095".
+                example: 19,23,21,4095
             compression:
-                type: scalar
+                type: str
                 desc: |
                     Type of compression to use when creating archives. See
                     https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
-                    Defaults to no compression.
+                    Defaults to "lz4".
                 example: lz4
             remote_rate_limit:
                 type: int
-                desc: Remote network upload rate limit in kiBytes/second.
+                desc: Remote network upload rate limit in kiBytes/second. Defaults to unlimited.
                 example: 100
             ssh_command:
-                type: scalar
-                desc: Command to use instead of just "ssh". This can be used to specify ssh options.
+                type: str
+                desc: |
+                    Command to use instead of "ssh". This can be used to specify ssh options.
+                    Defaults to not set.
                 example: ssh -i /path/to/private/key
+            borg_base_directory:
+                type: str
+                desc: |
+                    Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
+                    See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
+                example: /path/to/base
+            borg_config_directory:
+                type: str
+                desc: |
+                    Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
+                example: /path/to/base/config
+            borg_cache_directory:
+                type: str
+                desc: |
+                    Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
+                example: /path/to/base/cache
+            borg_security_directory:
+                type: str
+                desc: |
+                    Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
+                example: /path/to/base/config/security
+            borg_keys_directory:
+                type: str
+                desc: |
+                    Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
+                example: /path/to/base/config/keys
             umask:
                 type: scalar
-                desc: Umask to be used for borg create.
+                desc: Umask to be used for borg create. Defaults to 0077.
                 example: 0077
             lock_wait:
                 type: int
-                desc: Maximum seconds to wait for acquiring a repository/cache lock.
+                desc: Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1.
                 example: 5
             archive_name_format:
-                type: scalar
+                type: str
                 desc: |
                     Name of the archive. Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. Default is
+                    "borg help placeholders" for details. Defaults to
                     "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
                     also specify a prefix in the retention section to avoid accidental pruning of
                     archives with a different archive name format. And you should also specify a
                     prefix in the consistency section as well.
                 example: "{hostname}-documents-{now}"
+            relocated_repo_access_is_ok:
+                type: bool
+                desc: Bypass Borg error about a repository that has been moved. Defaults to false.
+                example: true
+            unknown_unencrypted_repo_access_is_ok:
+                type: bool
+                desc: |
+                    Bypass Borg error about a previously unknown unencrypted repository. Defaults to
+                    false.
+                example: true
     retention:
         desc: |
             Retention policy for how many backups to keep in each category. See
             https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
-            At least one of the "keep" options is required for pruning to work.
+            At least one of the "keep" options is required for pruning to work. See
+            https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/
+            if you'd like to skip pruning entirely.
         map:
             keep_within:
-                type: scalar
+                type: str
                 desc: Keep all archives within this time interval.
                 example: 3H
             keep_secondly:
@@ -210,11 +286,11 @@ map:
                 desc: Number of yearly archives to keep.
                 example: 1
             prefix:
-                type: scalar
+                type: str
                 desc: |
                     When pruning, only consider archive names starting with this prefix.
                     Borg placeholders can be used. See the output of "borg help placeholders" for
-                    details. Default is "{hostname}-".
+                    details. Defaults to "{hostname}-". Use an empty value to disable the default.
                 example: sourcehostname
     consistency:
         desc: |
@@ -225,20 +301,21 @@ map:
             checks:
                 seq:
                     - type: str
-                      enum: ['repository', 'archives', 'extract', 'disabled']
+                      enum: ['repository', 'archives', 'data', 'extract', 'disabled']
                       unique: true
                 desc: |
-                    List of one or more consistency checks to run: "repository", "archives", and/or
-                    "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
-                    all consistency checks. "repository" checks the consistency of the repository,
-                    "archive" checks all of the archives, and "extract" does an extraction dry-run
-                    of just the most recent archive.
+                    List of one or more consistency checks to run: "repository", "archives", "data",
+                    and/or "extract". Defaults to "repository" and "archives". Set to "disabled" to
+                    disable all consistency checks. "repository" checks the consistency of the
+                    repository, "archives" checks all of the archives, "data" verifies the integrity
+                    of the data within the archives, and "extract" does an extraction dry-run of the
+                    most recent archive. Note that "data" implies "archives".
                 example:
                     - repository
                     - archives
             check_repositories:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Paths to a subset of the repositories in the location section on which to run
                     consistency checks. Handy in case some of your repositories are very large, and
@@ -249,37 +326,207 @@ map:
             check_last:
                 type: int
                 desc: Restrict the number of checked archives to the last n. Applies only to the
-                      "archives" check.
+                      "archives" check. Defaults to checking all archives.
                 example: 3
             prefix:
-                type: scalar
+                type: str
                 desc: |
                     When performing the "archives" check, only consider archive names starting with
                     this prefix. Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. Default is "{hostname}-".
+                    "borg help placeholders" for details. Defaults to "{hostname}-". Use an empty
+                    value to disable the default.
                 example: sourcehostname
+    output:
+        desc: |
+            Options for customizing borgmatic's own output and logging.
+        map:
+            color:
+                type: bool
+                desc: |
+                    Apply color to console output. Can be overridden with --no-color command-line
+                    flag. Defaults to true.
+                example: false
     hooks:
         desc: |
-            Shell commands or scripts to execute before and after a backup or if an error has occurred.
-            IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
-            Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
-            prevent potential shell injection or privilege escalation.
+            Shell commands, scripts, or integrations to execute at various points during a borgmatic
+            run. IMPORTANT: All provided commands and scripts are executed with user permissions of
+            borgmatic. Do not forget to set secure permissions on this configuration file (chmod
+            0600) as well as on any script called from a hook (chmod 0700) to prevent potential
+            shell injection or privilege escalation.
         map:
             before_backup:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute before creating a backup.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute before creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "`date` - Starting a backup job."
+                    - echo "Starting a backup."
             after_backup:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute after creating a backup.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute after creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "`date` - Backup created."
+                    - echo "Created a backup."
             on_error:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute when an exception
+                    occurs during a backup or when running a before_backup or after_backup hook.
                 example:
-                    - echo "`date` - Error while creating a backup."
+                    - echo "Error while creating a backup or running a backup hook."
+            postgresql_databases:
+                seq:
+                    - map:
+                        name:
+                            required: true
+                            type: str
+                            desc: |
+                                Database name (required if using this hook). Or "all" to dump all
+                                databases on the host.
+                            example: users
+                        hostname:
+                            type: str
+                            desc: |
+                                Database hostname to connect to. Defaults to connecting via local
+                                Unix socket.
+                            example: database.example.org
+                        port:
+                            type: int
+                            desc: Port to connect to. Defaults to 5432.
+                            example: 5433
+                        username:
+                            type: str
+                            desc: |
+                                Username with which to connect to the database. Defaults to the
+                                username of the current user. You probably want to specify the
+                                "postgres" superuser here when the database name is "all".
+                            example: dbuser
+                        password:
+                            type: str
+                            desc: |
+                                Password with which to connect to the database. Omitting a password
+                                will only work if PostgreSQL is configured to trust the configured
+                                username without a password, or you create a ~/.pgpass file.
+                            example: trustsome1
+                        format:
+                            type: str
+                            enum: ['plain', 'custom', 'directory', 'tar']
+                            desc: |
+                                Database dump output format. One of "plain", "custom", "directory",
+                                or "tar". Defaults to "custom" (unlike raw pg_dump). See
+                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                                Note that format is ignored when the database name is "all".
+                            example: directory
+                        options:
+                            type: str
+                            desc: |
+                                Additional pg_dump/pg_dumpall options to pass directly to the dump
+                                command, without performing any validation on them. See
+                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                            example: --role=someone
+                desc: |
+                    List of one or more PostgreSQL databases to dump before creating a backup,
+                    run once per configuration file. The database dumps are added to your source
+                    directories at runtime, backed up, and then removed afterwards. Requires
+                    pg_dump/pg_dumpall/pg_restore commands. See
+                    https://www.postgresql.org/docs/current/app-pgdump.html for details.
+            mysql_databases:
+                seq:
+                    - map:
+                        name:
+                            required: true
+                            type: str
+                            desc: |
+                                Database name (required if using this hook). Or "all" to dump all
+                                databases on the host.
+                            example: users
+                        hostname:
+                            type: str
+                            desc: |
+                                Database hostname to connect to. Defaults to connecting via local
+                                Unix socket.
+                            example: database.example.org
+                        port:
+                            type: int
+                            desc: Port to connect to. Defaults to 3306.
+                            example: 3307
+                        username:
+                            type: str
+                            desc: |
+                                Username with which to connect to the database. Defaults to the
+                                username of the current user.
+                            example: dbuser
+                        password:
+                            type: str
+                            desc: |
+                                Password with which to connect to the database. Omitting a password
+                                will only work if MySQL is configured to trust the configured
+                                username without a password.
+                            example: trustsome1
+                        options:
+                            type: str
+                            desc: |
+                                Additional mysqldump options to pass directly to the dump command,
+                                without performing any validation on them. See
+                                https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
+                                https://mariadb.com/kb/en/library/mysqldump/ for details.
+                            example: --skip-comments
+                desc: |
+                    List of one or more MySQL/MariaDB databases to dump before creating a backup,
+                    run once per configuration file. The database dumps are added to your source
+                    directories at runtime, backed up, and then removed afterwards. Requires
+                    mysqldump/mysql commands (from either MySQL or MariaDB). See
+                    https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
+                    https://mariadb.com/kb/en/library/mysqldump/ for details.
+            healthchecks:
+                type: str
+                desc: |
+                    Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
+                    Create an account at https://healthchecks.io if you'd like to use this service.
+                    See
+                    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+                    for details.
+                example:
+                    https://hc-ping.com/your-uuid-here
+            cronitor:
+                type: str
+                desc: |
+                    Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
+                    account at https://cronitor.io if you'd like to use this service. See
+                    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+                    for details.
+                example:
+                    https://cronitor.link/d3x0c1
+            cronhub:
+                type: str
+                desc: |
+                    Cronhub ping URL to notify when a backup begins, ends, or errors. Create an
+                    account at https://cronhub.io if you'd like to use this service. See
+                    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook for
+                    details.
+                example:
+                    https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
+            before_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute before running all
+                    actions (if one of them is "create"), run once before all configuration files.
+                example:
+                    - echo "Starting actions."
+            after_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute after running all
+                    actions (if one of them is "create"), run once after all configuration files.
+                example:
+                    - echo "Completed actions."
+            umask:
+                type: scalar
+                desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.
+                example: 0077

borgmatic/config/validate.py

@@ -3,10 +3,9 @@ import logging
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-from ruamel import yaml
+import ruamel.yaml
 
-
-logger = logging.getLogger(__name__)
+from borgmatic.config import load
 
 
 def schema_filename():
@@ -64,12 +63,22 @@ def apply_logical_validation(config_filename, parsed_configuration):
                 ),
             )
 
-    consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
-    if archive_name_format and not consistency_prefix:
-        logger.warning(
-            'Since version 1.1.16, if you provide `archive_name_format`, you should also'
-            ' specify `consistency.prefix`.'
-        )
+
+def remove_examples(schema):
+    '''
+    pykwalify gets angry if the example field is not a string. So rather than bend to its will,
+    remove all examples from the given schema before passing the schema to pykwalify.
+    '''
+    if 'map' in schema:
+        for item_name, item_schema in schema['map'].items():
+            item_schema.pop('example', None)
+            remove_examples(item_schema)
+    elif 'seq' in schema:
+        for item_schema in schema['seq']:
+            item_schema.pop('example', None)
+            remove_examples(item_schema)
+
+    return schema
 
 
 def parse_configuration(config_filename, schema_filename):
@@ -87,18 +96,12 @@ def parse_configuration(config_filename, schema_filename):
     logging.getLogger('pykwalify').setLevel(logging.ERROR)
 
     try:
-        config = yaml.safe_load(open(config_filename))
-        schema = yaml.safe_load(open(schema_filename))
-    except yaml.error.YAMLError as error:
+        config = load.load_configuration(config_filename)
+        schema = load.load_configuration(schema_filename)
+    except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
-    # simply remove all examples before passing the schema to pykwalify.
-    for section_name, section_schema in schema['map'].items():
-        for field_name, field_schema in section_schema['map'].items():
-            field_schema.pop('example', None)
-
-    validator = pykwalify.core.Core(source_data=config, schema_data=schema)
+    validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
     parsed_result = validator.validate(raise_exception=False)
 
     if validator.validation_errors:
@@ -107,3 +110,46 @@ def parse_configuration(config_filename, schema_filename):
     apply_logical_validation(config_filename, parsed_result)
 
     return parsed_result
+
+
+def guard_configuration_contains_repository(repository, configurations):
+    '''
+    Given a repository path and a dict mapping from config filename to corresponding parsed config
+    dict, ensure that the repository is declared exactly once in all of the configurations.
+
+    If no repository is given, then error if there are multiple configured repositories.
+
+    Raise ValueError if the repository is not found in a configuration, or is declared multiple
+    times.
+    '''
+    if not repository:
+        count = len(
+            tuple(
+                config_repository
+                for config in configurations.values()
+                for config_repository in config['location']['repositories']
+            )
+        )
+
+        if count > 1:
+            raise ValueError(
+                'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
+                    repository
+                )
+            )
+
+        return
+
+    count = len(
+        tuple(
+            config_repository
+            for config in configurations.values()
+            for config_repository in config['location']['repositories']
+            if repository == config_repository
+        )
+    )
+
+    if count == 0:
+        raise ValueError('Repository {} not found in configuration files'.format(repository))
+    if count > 1:
+        raise ValueError('Repository {} found in multiple configuration files'.format(repository))

borgmatic/execute.py

@@ -0,0 +1,128 @@
+import logging
+import os
+import subprocess
+
+logger = logging.getLogger(__name__)
+
+
+ERROR_OUTPUT_MAX_LINE_COUNT = 25
+BORG_ERROR_EXIT_CODE = 2
+
+
+def exit_code_indicates_error(command, exit_code, error_on_warnings=False):
+    '''
+    Return True if the given exit code from running the command corresponds to an error.
+    '''
+    # If we're running something other than Borg, treat all non-zero exit codes as errors.
+    if 'borg' in command[0] and not error_on_warnings:
+        return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+
+    return bool(exit_code != 0)
+
+
+def log_output(command, process, output_buffer, output_log_level, error_on_warnings):
+    '''
+    Given a command already executed, its process opened by subprocess.Popen(), and the process'
+    relevant output buffer (stderr or stdout), log its output with the requested log level.
+    Additionally, raise a CalledProcessException if the process exits with an error (or a warning,
+    if error on warnings is True).
+    '''
+    last_lines = []
+
+    while process.poll() is None:
+        line = output_buffer.readline().rstrip().decode()
+        if not line:
+            continue
+
+        # Keep the last few lines of output in case the command errors, and we need the output for
+        # the exception below.
+        last_lines.append(line)
+        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.pop(0)
+
+        logger.log(output_log_level, line)
+
+    remaining_output = output_buffer.read().rstrip().decode()
+    if remaining_output:  # pragma: no cover
+        logger.log(output_log_level, remaining_output)
+
+    exit_code = process.poll()
+
+    if exit_code_indicates_error(command, exit_code, error_on_warnings):
+        # If an error occurs, include its output in the raised exception so that we don't
+        # inadvertently hide error output.
+        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.insert(0, '...')
+
+        raise subprocess.CalledProcessError(exit_code, ' '.join(command), '\n'.join(last_lines))
+
+
+def execute_command(
+    full_command,
+    output_log_level=logging.INFO,
+    output_file=None,
+    input_file=None,
+    shell=False,
+    extra_environment=None,
+    working_directory=None,
+    error_on_warnings=False,
+):
+    '''
+    Execute the given command (a sequence of command/argument strings) and log its output at the
+    given log level. If output log level is None, instead capture and return the output. If an
+    open output file object is given, then write stdout to the file and only log stderr (but only
+    if an output log level is set). If an open input file object is given, then read stdin from the
+    file. If shell is True, execute the command within a shell. If an extra environment dict is
+    given, then use it to augment the current environment, and pass the result into the command. If
+    a working directory is given, use that as the present working directory when running the
+    command.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command.
+    '''
+    logger.debug(
+        ' '.join(full_command)
+        + (' < {}'.format(input_file.name) if input_file else '')
+        + (' > {}'.format(output_file.name) if output_file else '')
+    )
+    environment = {**os.environ, **extra_environment} if extra_environment else None
+
+    if output_log_level is None:
+        output = subprocess.check_output(
+            full_command, shell=shell, env=environment, cwd=working_directory
+        )
+        return output.decode() if output is not None else None
+    else:
+        process = subprocess.Popen(
+            full_command,
+            stdin=input_file,
+            stdout=output_file or subprocess.PIPE,
+            stderr=subprocess.PIPE if output_file else subprocess.STDOUT,
+            shell=shell,
+            env=environment,
+            cwd=working_directory,
+        )
+        log_output(
+            full_command,
+            process,
+            process.stderr if output_file else process.stdout,
+            output_log_level,
+            error_on_warnings,
+        )
+
+
+def execute_command_without_capture(full_command, working_directory=None, error_on_warnings=False):
+    '''
+    Execute the given command (a sequence of command/argument strings), but don't capture or log its
+    output in any way. This is necessary for commands that monkey with the terminal (e.g. progress
+    display) or provide interactive prompts.
+
+    If a working directory is given, use that as the present working directory when running the
+    command.
+    '''
+    logger.debug(' '.join(full_command))
+
+    try:
+        subprocess.check_call(full_command, cwd=working_directory)
+    except subprocess.CalledProcessError as error:
+        if exit_code_indicates_error(full_command, error.returncode, error_on_warnings):
+            raise

borgmatic/hooks/command.py

@@ -0,0 +1,71 @@
+import logging
+import os
+
+from borgmatic import execute
+
+logger = logging.getLogger(__name__)
+
+
+def interpolate_context(command, context):
+    '''
+    Given a single hook command and a dict of context names/values, interpolate the values by
+    "{name}" into the command and return the result.
+    '''
+    for name, value in context.items():
+        command = command.replace('{%s}' % name, str(value))
+
+    return command
+
+
+def execute_hook(commands, umask, config_filename, description, dry_run, **context):
+    '''
+    Given a list of hook commands to execute, a umask to execute with (or None), a config filename,
+    a hook description, and whether this is a dry run, run the given commands. Or, don't run them
+    if this is a dry run.
+
+    The context contains optional values interpolated by name into the hook commands. Currently,
+    this only applies to the on_error hook.
+
+    Raise ValueError if the umask cannot be parsed.
+    Raise subprocesses.CalledProcessError if an error occurs in a hook.
+    '''
+    if not commands:
+        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
+        return
+
+    dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
+
+    context['configuration_filename'] = config_filename
+    commands = [interpolate_context(command, context) for command in commands]
+
+    if len(commands) == 1:
+        logger.info(
+            '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
+        )
+    else:
+        logger.info(
+            '{}: Running {} commands for {} hook{}'.format(
+                config_filename, len(commands), description, dry_run_label
+            )
+        )
+
+    if umask:
+        parsed_umask = int(str(umask), 8)
+        logger.debug('{}: Set hook umask to {}'.format(config_filename, oct(parsed_umask)))
+        original_umask = os.umask(parsed_umask)
+    else:
+        original_umask = None
+
+    try:
+        for command in commands:
+            if not dry_run:
+                execute.execute_command(
+                    [command],
+                    output_log_level=logging.ERROR
+                    if description == 'on-error'
+                    else logging.WARNING,
+                    shell=True,
+                )
+    finally:
+        if original_umask:
+            os.umask(original_umask)

borgmatic/hooks/cronhub.py

@@ -0,0 +1,26 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_cronhub(ping_url, config_filename, dry_run, state):
+    '''
+    Ping the given Cronhub URL, substituting in the state string. Use the given configuration
+    filename in any log entries. If this is a dry run, then don't actually ping anything.
+    '''
+    if not ping_url:
+        logger.debug('{}: No Cronhub hook set'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    formatted_state = '/{}/'.format(state)
+    ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
+
+    logger.info('{}: Pinging Cronhub {}{}'.format(config_filename, state, dry_run_label))
+    logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url))
+
+    if not dry_run:
+        logging.getLogger('urllib3').setLevel(logging.ERROR)
+        requests.get(ping_url)

borgmatic/hooks/cronitor.py

@@ -0,0 +1,25 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_cronitor(ping_url, config_filename, dry_run, append):
+    '''
+    Ping the given Cronitor URL, appending the append string. Use the given configuration filename
+    in any log entries. If this is a dry run, then don't actually ping anything.
+    '''
+    if not ping_url:
+        logger.debug('{}: No Cronitor hook set'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
+    logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
+
+    if not dry_run:
+        logging.getLogger('urllib3').setLevel(logging.ERROR)
+        requests.get(ping_url)

borgmatic/hooks/dispatch.py

@@ -0,0 +1,55 @@
+import logging
+
+from borgmatic.hooks import mysql, postgresql
+
+logger = logging.getLogger(__name__)
+
+HOOK_NAME_TO_MODULE = {'postgresql_databases': postgresql, 'mysql_databases': mysql}
+
+
+def call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs):
+    '''
+    Given the hooks configuration dict and a prefix to use in log entries, call the requested
+    function of the Python module corresponding to the given hook name. Supply that call with the
+    configuration for this hook, the log prefix, and any given args and kwargs. Return any return
+    value.
+
+    If the hook name is not present in the hooks configuration, then bail without calling anything.
+
+    Raise ValueError if the hook name is unknown.
+    Raise AttributeError if the function name is not found in the module.
+    Raise anything else that the called function raises.
+    '''
+    config = hooks.get(hook_name)
+    if not config:
+        logger.debug('{}: No {} hook configured.'.format(log_prefix, hook_name))
+        return
+
+    try:
+        module = HOOK_NAME_TO_MODULE[hook_name]
+    except KeyError:
+        raise ValueError('Unknown hook name: {}'.format(hook_name))
+
+    logger.debug('{}: Calling {} hook function {}'.format(log_prefix, hook_name, function_name))
+    return getattr(module, function_name)(config, log_prefix, *args, **kwargs)
+
+
+def call_hooks(function_name, hooks, log_prefix, hook_names, *args, **kwargs):
+    '''
+    Given the hooks configuration dict and a prefix to use in log entries, call the requested
+    function of the Python module corresponding to each given hook name. Supply each call with the
+    configuration for that hook, the log prefix, and any given args and kwargs. Collect any return
+    values into a dict from hook name to return value.
+
+    If the hook name is not present in the hooks configuration, then don't call the function for it,
+    and omit it from the return values.
+
+    Raise ValueError if the hook name is unknown.
+    Raise AttributeError if the function name is not found in the module.
+    Raise anything else that a called function raises. An error stops calls to subsequent functions.
+    '''
+    return {
+        hook_name: call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs)
+        for hook_name in hook_names
+        if hook_name in hooks
+    }

borgmatic/hooks/dump.py

@@ -0,0 +1,138 @@
+import glob
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases')
+
+
+def make_database_dump_filename(dump_path, name, hostname=None):
+    '''
+    Based on the given dump directory path, database name, and hostname, return a filename to use
+    for the database dump. The hostname defaults to localhost.
+
+    Raise ValueError if the database name is invalid.
+    '''
+    if os.path.sep in name:
+        raise ValueError('Invalid database name {}'.format(name))
+
+    return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
+
+
+def remove_database_dumps(dump_path, databases, database_type_name, log_prefix, dry_run):
+    '''
+    Remove the database dumps for the given databases in the dump directory path. The databases are
+    supplied as a sequence of dicts, one dict describing each database as per the configuration
+    schema. Use the name of the database type and the log prefix in any log entries. If this is a
+    dry run, then don't actually remove anything.
+    '''
+    if not databases:
+        logger.debug('{}: No {} databases configured'.format(log_prefix, database_type_name))
+        return
+
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    logger.info(
+        '{}: Removing {} database dumps{}'.format(log_prefix, database_type_name, dry_run_label)
+    )
+
+    for database in databases:
+        dump_filename = make_database_dump_filename(
+            dump_path, database['name'], database.get('hostname')
+        )
+
+        logger.debug(
+            '{}: Removing {} database dump {} from {}{}'.format(
+                log_prefix, database_type_name, database['name'], dump_filename, dry_run_label
+            )
+        )
+        if dry_run:
+            continue
+
+        os.remove(dump_filename)
+        dump_path = os.path.dirname(dump_filename)
+
+        if len(os.listdir(dump_path)) == 0:
+            os.rmdir(dump_path)
+
+
+def convert_glob_patterns_to_borg_patterns(patterns):
+    '''
+    Convert a sequence of shell glob patterns like "/etc/*" to the corresponding Borg archive
+    patterns like "sh:etc/*".
+    '''
+    return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns]
+
+
+def get_database_names_from_dumps(patterns):
+    '''
+    Given a sequence of database dump patterns, find the corresponding database dumps on disk and
+    return the database names from their filenames.
+    '''
+    return [os.path.basename(dump_path) for pattern in patterns for dump_path in glob.glob(pattern)]
+
+
+def get_database_configurations(databases, names):
+    '''
+    Given the full database configuration dicts as per the configuration schema, and a sequence of
+    database names, filter down and yield the configuration for just the named databases.
+    Additionally, if a database configuration is named "all", project out that configuration for
+    each named database.
+    '''
+    named_databases = {database['name']: database for database in databases}
+
+    for name in names:
+        database = named_databases.get(name)
+        if database:
+            yield database
+            continue
+
+        if 'all' in named_databases:
+            yield {**named_databases['all'], **{'name': name}}
+            continue
+
+
+def get_per_hook_database_configurations(hooks, names, dump_patterns):
+    '''
+    Given the hooks configuration dict as per the configuration schema, a sequence of database
+    names to restore, and a dict from database hook name to glob patterns for matching dumps,
+    filter down the configuration for just the named databases.
+
+    If there are no named databases given, then find the corresponding database dumps on disk and
+    use the database names from their filenames. Additionally, if a database configuration is named
+    "all", project out that configuration for each named database.
+
+    Return the results as a dict from database hook name to a sequence of database configuration
+    dicts for that database type.
+
+    Raise ValueError if one of the database names cannot be matched to a database in borgmatic's
+    database configuration.
+    '''
+    # TODO: Need to filter names by database type? Maybe take a database --type argument to disambiguate.
+    hook_databases = {
+        hook_name: list(
+            get_database_configurations(
+                hooks.get(hook_name),
+                names or get_database_names_from_dumps(dump_patterns[hook_name]),
+            )
+        )
+        for hook_name in DATABASE_HOOK_NAMES
+        if hook_name in hooks
+    }
+
+    if not names or 'all' in names:
+        return hook_databases
+
+    found_names = {
+        database['name'] for databases in hook_databases.values() for database in databases
+    }
+    missing_names = sorted(set(names) - found_names)
+    if missing_names:
+        raise ValueError(
+            'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
+                ', '.join(missing_names)
+            )
+        )
+
+    return hook_databases

borgmatic/hooks/healthchecks.py

@@ -0,0 +1,37 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
+    '''
+    Ping the given Healthchecks URL or UUID, appending the append string if any. Use the given
+    configuration filename in any log entries. If this is a dry run, then don't actually ping
+    anything.
+    '''
+    if not ping_url_or_uuid:
+        logger.debug('{}: No Healthchecks hook set'.format(config_filename))
+        return
+
+    ping_url = (
+        ping_url_or_uuid
+        if ping_url_or_uuid.startswith('http')
+        else 'https://hc-ping.com/{}'.format(ping_url_or_uuid)
+    )
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+    if append:
+        ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info(
+        '{}: Pinging Healthchecks{}{}'.format(
+            config_filename, ' ' + append if append else '', dry_run_label
+        )
+    )
+    logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
+
+    if not dry_run:
+        logging.getLogger('urllib3').setLevel(logging.ERROR)
+        requests.get(ping_url)

borgmatic/hooks/mysql.py

@@ -0,0 +1,94 @@
+import logging
+import os
+
+from borgmatic.execute import execute_command
+from borgmatic.hooks import dump
+
+DUMP_PATH = '~/.borgmatic/mysql_databases'
+logger = logging.getLogger(__name__)
+
+
+def dump_databases(databases, log_prefix, dry_run):
+    '''
+    Dump the given MySQL/MariaDB databases to disk. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given log
+    prefix in any log entries. If this is a dry run, then don't actually dump anything.
+    '''
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+
+    logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
+
+    for database in databases:
+        name = database['name']
+        dump_filename = dump.make_database_dump_filename(DUMP_PATH, name, database.get('hostname'))
+        command = (
+            ('mysqldump', '--add-drop-database')
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+            + (('--user', database['username']) if 'username' in database else ())
+            + (tuple(database['options'].split(' ')) if 'options' in database else ())
+            + (('--all-databases',) if name == 'all' else ('--databases', name))
+        )
+        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+
+        logger.debug(
+            '{}: Dumping MySQL database {} to {}{}'.format(
+                log_prefix, name, dump_filename, dry_run_label
+            )
+        )
+        if not dry_run:
+            os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
+            execute_command(
+                command, output_file=open(dump_filename, 'w'), extra_environment=extra_environment
+            )
+
+
+def remove_database_dumps(databases, log_prefix, dry_run):  # pragma: no cover
+    '''
+    Remove the database dumps for the given databases. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the log prefix in
+    any log entries. If this is a dry run, then don't actually remove anything.
+    '''
+    dump.remove_database_dumps(DUMP_PATH, databases, 'MySQL', log_prefix, dry_run)
+
+
+def make_database_dump_patterns(databases, log_prefix, names):
+    '''
+    Given a sequence of configurations dicts, a prefix to log with, and a sequence of database
+    names to match, return the corresponding glob patterns to match the database dumps in an
+    archive. An empty sequence of names indicates that the patterns should match all dumps.
+    '''
+    return [
+        dump.make_database_dump_filename(DUMP_PATH, name, hostname='*') for name in (names or ['*'])
+    ]
+
+
+def restore_database_dumps(databases, log_prefix, dry_run):
+    '''
+    Restore the given MySQL/MariaDB databases from disk. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given log
+    prefix in any log entries. If this is a dry run, then don't actually restore anything.
+    '''
+    dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+
+    for database in databases:
+        dump_filename = dump.make_database_dump_filename(
+            DUMP_PATH, database['name'], database.get('hostname')
+        )
+        restore_command = (
+            ('mysql', '--batch')
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+            + (('--user', database['username']) if 'username' in database else ())
+        )
+        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+
+        logger.debug(
+            '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
+        )
+        if not dry_run:
+            execute_command(
+                restore_command, input_file=open(dump_filename), extra_environment=extra_environment
+            )

borgmatic/hooks/postgresql.py

@@ -0,0 +1,104 @@
+import logging
+import os
+
+from borgmatic.execute import execute_command
+from borgmatic.hooks import dump
+
+DUMP_PATH = '~/.borgmatic/postgresql_databases'
+logger = logging.getLogger(__name__)
+
+
+def dump_databases(databases, log_prefix, dry_run):
+    '''
+    Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
+    one dict describing each database as per the configuration schema. Use the given log prefix in
+    any log entries. If this is a dry run, then don't actually dump anything.
+    '''
+    dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+
+    logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label))
+
+    for database in databases:
+        name = database['name']
+        dump_filename = dump.make_database_dump_filename(DUMP_PATH, name, database.get('hostname'))
+        all_databases = bool(name == 'all')
+        command = (
+            ('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
+            + ('--file', dump_filename)
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + (() if all_databases else ('--format', database.get('format', 'custom')))
+            + (tuple(database['options'].split(' ')) if 'options' in database else ())
+            + (() if all_databases else (name,))
+        )
+        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+
+        logger.debug(
+            '{}: Dumping PostgreSQL database {} to {}{}'.format(
+                log_prefix, name, dump_filename, dry_run_label
+            )
+        )
+        if not dry_run:
+            os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
+            execute_command(command, extra_environment=extra_environment)
+
+
+def remove_database_dumps(databases, log_prefix, dry_run):  # pragma: no cover
+    '''
+    Remove the database dumps for the given databases. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the log prefix in
+    any log entries. If this is a dry run, then don't actually remove anything.
+    '''
+    dump.remove_database_dumps(DUMP_PATH, databases, 'PostgreSQL', log_prefix, dry_run)
+
+
+def make_database_dump_patterns(databases, log_prefix, names):
+    '''
+    Given a sequence of configurations dicts, a prefix to log with, and a sequence of database
+    names to match, return the corresponding glob patterns to match the database dumps in an
+    archive. An empty sequence of names indicates that the patterns should match all dumps.
+    '''
+    return [
+        dump.make_database_dump_filename(DUMP_PATH, name, hostname='*') for name in (names or ['*'])
+    ]
+
+
+def restore_database_dumps(databases, log_prefix, dry_run):
+    '''
+    Restore the given PostgreSQL databases from disk. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given log
+    prefix in any log entries. If this is a dry run, then don't actually restore anything.
+    '''
+    dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
+
+    for database in databases:
+        dump_filename = dump.make_database_dump_filename(
+            DUMP_PATH, database['name'], database.get('hostname')
+        )
+        restore_command = (
+            ('pg_restore', '--no-password', '--clean', '--if-exists', '--exit-on-error')
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + ('--dbname', database['name'])
+            + (dump_filename,)
+        )
+        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+        analyze_command = (
+            ('psql', '--no-password', '--quiet')
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + ('--dbname', database['name'])
+            + ('--command', 'ANALYZE')
+        )
+
+        logger.debug(
+            '{}: Restoring PostgreSQL database {}{}'.format(
+                log_prefix, database['name'], dry_run_label
+            )
+        )
+        if not dry_run:
+            execute_command(restore_command, extra_environment=extra_environment)
+            execute_command(analyze_command, extra_environment=extra_environment)

borgmatic/logger.py

@@ -0,0 +1,116 @@
+import logging
+import os
+import sys
+
+import colorama
+
+
+def to_bool(arg):
+    '''
+    Return a boolean value based on `arg`.
+    '''
+    if arg is None or isinstance(arg, bool):
+        return arg
+
+    if isinstance(arg, str):
+        arg = arg.lower()
+
+    if arg in ('yes', 'on', '1', 'true', 1):
+        return True
+
+    return False
+
+
+def interactive_console():
+    '''
+    Return whether the current console is "interactive". Meaning: Capable of
+    user input and not just something like a cron job.
+    '''
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+
+
+def should_do_markup(no_color, configs):
+    '''
+    Given the value of the command-line no-color argument, and a dict of configuration filename to
+    corresponding parsed configuration, determine if we should enable colorama marking up.
+    '''
+    if no_color:
+        return False
+
+    if any(config.get('output', {}).get('color') is False for config in configs.values()):
+        return False
+
+    py_colors = os.environ.get('PY_COLORS', None)
+
+    if py_colors is not None:
+        return to_bool(py_colors)
+
+    return interactive_console()
+
+
+LOG_LEVEL_TO_COLOR = {
+    logging.CRITICAL: colorama.Fore.RED,
+    logging.ERROR: colorama.Fore.RED,
+    logging.WARN: colorama.Fore.YELLOW,
+    logging.INFO: colorama.Fore.GREEN,
+    logging.DEBUG: colorama.Fore.CYAN,
+}
+
+
+class Console_color_formatter(logging.Formatter):
+    def format(self, record):
+        color = LOG_LEVEL_TO_COLOR.get(record.levelno)
+        return color_text(color, record.msg)
+
+
+def color_text(color, message):
+    '''
+    Give colored text.
+    '''
+    if not color:
+        return message
+
+    return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
+
+
+def configure_logging(
+    console_log_level, syslog_log_level=None, log_file_log_level=None, log_file=None
+):
+    '''
+    Configure logging to go to both the console and (syslog or log file). Use the given log levels,
+    respectively.
+
+    Raise FileNotFoundError or PermissionError if the log file could not be opened for writing.
+    '''
+    if syslog_log_level is None:
+        syslog_log_level = console_log_level
+    if log_file_log_level is None:
+        log_file_log_level = console_log_level
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(Console_color_formatter())
+    console_handler.setLevel(console_log_level)
+
+    syslog_path = None
+    if log_file is None:
+        if os.path.exists('/dev/log'):
+            syslog_path = '/dev/log'
+        elif os.path.exists('/var/run/syslog'):
+            syslog_path = '/var/run/syslog'
+
+    if syslog_path and not interactive_console():
+        syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
+        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
+        syslog_handler.setLevel(syslog_log_level)
+        handlers = (console_handler, syslog_handler)
+    elif log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s'))
+        file_handler.setLevel(log_file_log_level)
+        handlers = (console_handler, file_handler)
+    else:
+        handlers = (console_handler,)
+
+    logging.basicConfig(
+        level=min(console_log_level, syslog_log_level, log_file_log_level), handlers=handlers
+    )

borgmatic/verbosity.py

@@ -1,6 +1,6 @@
 import logging
 
-
+VERBOSITY_ERROR = -1
 VERBOSITY_WARNING = 0
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2
@@ -11,6 +11,7 @@ def verbosity_to_log_level(verbosity):
     Given a borgmatic verbosity value, return the corresponding Python log level.
     '''
     return {
+        VERBOSITY_ERROR: logging.ERROR,
         VERBOSITY_WARNING: logging.WARNING,
         VERBOSITY_SOME: logging.INFO,
         VERBOSITY_LOTS: logging.DEBUG,

docs/Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.7.4-alpine3.10 as borgmatic
+
+COPY . /app
+RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
+RUN borgmatic --help > /command-line.txt \
+    && for action in init prune create check extract restore list info; do \
+           echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
+           && borgmatic "$action" --help >> /command-line.txt; done
+
+FROM node:12.10.0-alpine as html
+
+ARG ENVIRONMENT=production
+
+WORKDIR /source
+
+RUN npm install @11ty/eleventy \
+    @11ty/eleventy-plugin-syntaxhighlight \
+    @11ty/eleventy-plugin-inclusive-language \
+    markdown-it \
+    markdown-it-anchor \
+    markdown-it-replace-link
+COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml
+COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt
+COPY . /source
+RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
+  && mv /output/docs/index.html /output/index.html
+
+FROM nginx:1.16.1-alpine
+
+COPY --from=html /output /usr/share/nginx/html
+COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/README.md

@@ -0,0 +1 @@
+../README.md

docs/_data/layout.json

@@ -0,0 +1 @@
+"layouts/main.njk"

docs/_includes/asciinema.css

@@ -0,0 +1,3 @@
+.asciicast > iframe {
+    width: 100% !important;
+}

docs/_includes/components/external-links.css

@@ -0,0 +1,12 @@
+/* External links */
+a[href^="http://"]:not(.minilink):not(.elv-externalexempt),
+a[href^="https://"]:not(.minilink):not(.elv-externalexempt), 
+a[href^="//"]:not(.minilink):not(.elv-externalexempt) {
+	text-decoration-color: inherit;
+}
+/* External link hovers */
+a[href^="http://"]:not(.minilink):not(.elv-externalexempt):hover,
+a[href^="https://"]:not(.minilink):not(.elv-externalexempt):hover, 
+a[href^="//"]:not(.minilink):not(.elv-externalexempt):hover {
+	text-decoration-color: #00bcd4;
+}

docs/_includes/components/info-blocks.css

@@ -0,0 +1,34 @@
+/* Warning */
+.elv-info {
+	line-height: 1.5;
+	padding: 0.8125em 1em 0.75em; /* 13px 16px 12px /16 */
+	margin-left: -1rem;
+	margin-right: -1rem;
+	margin-bottom: 2em;
+	background-color: #dff7ff;
+}
+.elv-info:before {
+	content: "ℹ️ ";
+}
+.elv-info-warn {
+	background-color: #ffa;
+}
+.elv-info-warn:before {
+	content: "⚠️ ";
+}
+.elv-info:first-child {
+	margin-top: 0;
+}
+body > .elv-info {
+	margin-left: 0;
+	margin-right: 0;
+	padding: .5rem 1rem;
+}
+@media (min-width: 37.5em) and (min-height: 25em) { /* 600px / 400px */
+	body > .elv-info-sticky {
+		position: sticky;
+		top: 0;
+		z-index: 2;
+		box-shadow: 0 3px 0 0 rgba(0,0,0,.08);
+	}
+}

docs/_includes/components/lists.css

@@ -0,0 +1,126 @@
+/* Buzzwords */
+@keyframes rainbow {
+	0% { background-position: 0% 50%; }
+	50% { background-position: 100% 50%; }
+	100% { background-position: 0% 50%; }
+}
+.buzzword-list,
+.inlinelist {
+	padding: 0;
+}
+.inlinelist:first-child:last-child {
+	margin: 0;
+}
+.buzzword,
+.buzzword-list li,
+.inlinelist .inlinelist-item {
+	display: inline;
+	-webkit-box-decoration-break: clone;
+	box-decoration-break: clone;
+	font-family: Georgia, serif;
+	font-size: 116%;
+	white-space: normal;
+	line-height: 1.85;
+	padding: .2em .5em;
+	margin: 4px 4px 4px 0;
+	transition: .15s linear outline;
+}
+.inlinelist .inlinelist-item.active {
+	background-color: #222;
+	color: #fff;
+	font-weight: inherit;
+}
+.inlinelist .inlinelist-item.active :link,
+.inlinelist .inlinelist-item.active :visited {
+	color: #fff;
+}
+.inlinelist .inlinelist-item code {
+	background-color: transparent;
+}
+a.buzzword {
+	text-decoration: underline;
+}
+.buzzword-list a,
+.inlinelist a {
+	text-decoration: none;
+}
+.inlinelist .inlinelist-item {
+	font-size: 100%;
+	line-height: 2;
+}
+@supports not(-webkit-box-decoration-break: clone) {
+	.buzzword,
+	.buzzword-list li,
+	.inlinelist .inlinelist-item {
+		display: inline-block;
+	}
+}
+.buzzword-list li,
+.buzzword {
+	background-color: #f7f7f7;
+}
+.inlinelist .inlinelist-item {
+	background-color: #e9e9e9;
+}
+.inlinelist .inlinelist-item:hover,
+.inlinelist .inlinelist-item:focus,
+.buzzword-list li:hover,
+.buzzword-list li:focus,
+.buzzword:hover,
+.buzzword:focus {
+	position: relative;
+	background-image: linear-gradient(238deg, #ff0000, #ff8000, #ffff00, #80ff00, #00ff00, #00ff80, #00ffff, #0080ff, #0000ff, #8000ff, #ff0080);
+	background-size: 1200% 1200%;
+	color: #fff;
+	text-shadow: 0 0 2px rgba(0,0,0,.9);
+	animation: rainbow 1.6s infinite;
+}
+.inlinelist .inlinelist-item:hover a,
+.inlinelist .inlinelist-item:focus a,
+.buzzword-list li:hover a,
+.buzzword-list li:focus a,
+a.buzzword:hover,
+a.buzzword:focus {
+	color: #fff;
+	text-decoration: none;
+}
+/*
+I wish there were a PE friendly way to do this but media queries don’t work work with @supports
+@media (prefers-reduced-motion: no-preference) {
+	.buzzword:hover,
+	.buzzword:focus {
+		animation: rainbow 1s infinite;
+	}
+}*/
+.buzzword-list li:hover:after,
+.buzzword-list li:focus:after,
+.buzzword:hover:after,
+.buzzword:focus:after {
+	font-family: system-ui, sans-serif;
+	content: "Buzzword alert!!!";
+	position: absolute;
+	left: 0;
+	top: 0;
+	max-width: 8em;
+	color: #f00;
+	font-weight: 700;
+	text-transform: uppercase;
+	transform: rotate(-10deg) translate(-25%, -125%);
+	text-shadow: 1px 1px 5px rgba(0,0,0,.6);
+	line-height: 1.2;
+	pointer-events: none;
+}
+main h2 .buzzword,
+main h3 .buzzword,
+main p .buzzword {
+	padding: 0px 7px;
+	font-size: 1em; /* 18px /18 */
+	margin: 0;
+	line-height: 1.444444444444; /* 26px /18 */
+	font-family: inherit;
+}
+main h2 a.buzzword,
+main h3 a.buzzword,
+main p a.buzzword {
+	text-decoration: underline;
+}

docs/_includes/components/minilink.css

@@ -0,0 +1,40 @@
+/* Mini link */
+.minilink {
+	display: inline-block;
+	padding: .125em .375em;
+	text-transform: uppercase;
+	font-size: 0.875rem; /* 14px /16 */
+	text-decoration: none;
+	background-color: #ddd;
+	border-radius: 0.1875em; /* 3px /16 */
+	font-weight: 500;
+	margin: 0 0.4285714285714em 0.07142857142857em 0; /* 0 6px 1px 0 /14 */
+	line-height: 1.285714285714; /* 18px /14 */
+	font-family: system-ui, sans-serif;
+}
+.minilink[href] {
+	box-shadow: 0 1px 1px 0 rgba(0,0,0,.5);
+}
+.minilink[href]:hover,
+.minilink[href]:focus {
+	background-color: #bbb;
+}
+pre + .minilink {
+	color: #fff;
+	border-radius: 0 0 0.2857142857143em 0.2857142857143em; /* 4px /14 */
+	float: right;
+	background-color: #444;
+	color: #fff;
+}
+pre[class*=language-] + .minilink {
+	position: relative;
+	top: -0.7142857142857em; /* -10px /14 */
+}
+p.minilink {
+	float: right;
+	margin-left: 2em;
+	margin-bottom: 2em;
+}
+.minilink + pre[class*=language-] {
+	clear: both;
+}

docs/_includes/components/suggestion-form.css

@@ -0,0 +1,18 @@
+#suggestion-form textarea {
+	font-family: sans-serif;
+	width: 100%;
+}
+
+#suggestion-form label {
+	font-weight: bold;
+}
+
+#suggestion-form input[type=email] {
+	font-size: 16px;
+	width: 100%;
+}
+
+#suggestion-form .form-error {
+        color: red;
+}
+

docs/_includes/components/suggestion-form.html

@@ -0,0 +1,33 @@
+<h2>Improve this documentation</h2>
+
+<p>Have an idea on how to make this documentation even better? Send your
+feedback below! (But if you need help installing or using borgmatic, please
+use our <a href="https://torsion.org/borgmatic/#issues">issue tracker</a>
+instead.)</p>
+
+<form id="suggestion-form">
+    <div><label for="suggestion">Suggestion</label></div>
+    <textarea id="suggestion" rows="8" cols="60" name="suggestion"></textarea>
+    <div data-sk-error="suggestion" class="form-error"></div>
+    <input id="_page" type="hidden" name="_page">
+    <input id="_subject" type="hidden" name="_subject" value="borgmatic documentation suggestion">
+    <br />
+    <label for="email">Email address</label>
+    <div><input id="email" type="email" name="email" placeholder="Only required if you want a response!"></div>
+    <div data-sk-error="email" class="form-error"></div>
+    <br />
+    <div><button type="submit">Send</button></div>
+    <br />
+</form>
+
+<script>
+    document.getElementById('_page').value = window.location.href;
+    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
+  
+    sk('form', 'init', {
+        id: '1d536680ab96',
+        element: '#suggestion-form'
+    });
+</script>
+
+<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/components/toc.css

@@ -0,0 +1,63 @@
+.elv-toc {
+	font-size: 1rem; /* Reset */
+}
+@media (min-width: 64em) { /* 1024px */
+	.elv-toc {
+		position: absolute;
+		left: -17rem;
+		width: 16rem;
+	}
+}
+
+
+.elv-toc-list {
+	padding-left: 0;
+	padding-right: 0;
+	list-style: none;
+}
+/* Nested lists */
+.elv-toc-list ul {
+	padding: 0;
+	display: none;
+	margin-bottom: 1.5em;
+	list-style: none;
+}
+.elv-toc-list ul li {
+	padding-left: 0.875em; /* 14px /16 */
+}
+@media (min-width: 64em) and (min-height: 48em) { /* 1024 x 768px */
+	.elv-toc-list ul {
+		display: block;
+	}
+}
+
+/* List items */
+.elv-toc-list a:not(:hover) {
+	text-decoration: none;
+}
+.elv-toc-list li {
+	padding-top: 0;
+	padding-bottom: 0;
+	margin: .1em 0 .5em;
+}
+/* Top level links */
+.elv-toc-list > li > a {
+	font-weight: 400;
+	font-size: 1.0625em; /* 17px /16 */
+	color: #222;
+}
+
+/* Active links */
+.elv-toc-list li.elv-toc-active > a {
+	font-weight: 700;
+	text-decoration: underline;
+}
+.elv-toc-active > a:after {
+	content: " ⬅";
+	line-height: .5;
+}
+/* Show only active nested lists */
+.elv-toc-list ul.elv-toc-active,
+.elv-toc-list li.elv-toc-active > ul {
+	display: block;
+}

docs/_includes/header.njk

@@ -0,0 +1,4 @@
+<header class="elv-layout elv-layout-full elv-header{% if headerClass %} {{ headerClass }}{% endif %}">
+    {% if page.url != '/' %}<h3><a href="https://torsion.org/borgmatic/">borgmatic</a></h3>{% endif %}
+    <h1 class="elv-hed">{{ title | safe }}</h1>
+</header>

docs/_includes/layouts/base.njk

@@ -0,0 +1,28 @@
+<!doctype html>
+<html lang="en"{% if templateClass %} class="{{ templateClass }}"{% endif %}>
+	<head>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<title>{{ subtitle + ' - ' if subtitle}}{{ title }}</title>
+{%- set css %}
+{% include 'index.css' %}
+{% include 'components/lists.css' %}
+{% include 'components/external-links.css' %}
+{% include 'components/minilink.css' %}
+{% include 'components/toc.css' %}
+{% include 'components/info-blocks.css' %}
+{% include 'components/suggestion-form.css' %}
+{% include 'prism-theme.css' %}
+{% include 'asciinema.css' %}
+{% endset %}
+		<style>{{ css | safe }}</style>
+{% if feedTitle and feedUrl %}
+		<link rel="alternate" href="{{ feedUrl }}" title="{{ feedTitle }}" type="application/atom+xml">
+{% endif %}
+	</head>
+	<body>
+
+		{{ content | safe }}
+
+	</body>
+</html>

docs/_includes/layouts/main.njk

@@ -0,0 +1,14 @@
+---
+layout: layouts/base.njk
+templateClass: elv-default
+headerClass: elv-header-default
+---
+{% include "header.njk" %}
+
+<main class="elv-layout{% if layoutClass %} {{ layoutClass }}{% endif %}">
+    <article>
+        {{ content | safe }}
+
+        {% include 'components/suggestion-form.html' %}
+    </article>
+</main>

docs/_includes/prism-theme.css

@@ -0,0 +1,172 @@
+/**
+ * prism.js default theme for JavaScript, CSS and HTML
+ * Based on dabblet (http://dabblet.com)
+ * @author Lea Verou
+ */
+code[class*="language-"],
+pre[class*="language-"] {
+  color: #ABB2BF;
+  background: none;
+  font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
+  text-align: left;
+  white-space: pre;
+  word-spacing: normal;
+  word-break: normal;
+  word-wrap: normal;
+  line-height: 1.5;
+  -moz-tab-size: 4;
+  -o-tab-size: 4;
+  tab-size: 4;
+  -webkit-hyphens: none;
+  -moz-hyphens: none;
+  -ms-hyphens: none;
+  hyphens: none;
+}
+
+pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection,
+code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection {
+  text-shadow: none;
+  background: #383e49;
+}
+
+pre[class*="language-"]::selection, pre[class*="language-"] ::selection,
+code[class*="language-"]::selection, code[class*="language-"] ::selection {
+  text-shadow: none;
+  background: #9aa2b1;
+}
+
+@media print {
+  code[class*="language-"],
+  pre[class*="language-"] {
+    text-shadow: none;
+  }
+}
+/* Code blocks */
+pre[class*="language-"] {
+  padding: 1em;
+  margin: .5em 0;
+  overflow: auto;
+}
+
+:not(pre) > code[class*="language-"],
+pre[class*="language-"] {
+  background: #282c34;
+}
+
+/* Inline code */
+:not(pre) > code[class*="language-"] {
+  padding: .1em;
+  border-radius: .3em;
+  white-space: normal;
+}
+
+.token.comment,
+.token.prolog,
+.token.doctype,
+.token.cdata {
+  color: #5C6370;
+}
+
+.token.punctuation {
+  color: #abb2bf;
+}
+
+.token.selector,
+.token.tag {
+  color: #e06c75;
+}
+
+.token.property,
+.token.boolean,
+.token.number,
+.token.constant,
+.token.symbol,
+.token.attr-name,
+.token.deleted {
+  color: #d19a66;
+}
+
+.token.string,
+.token.char,
+.token.attr-value,
+.token.builtin,
+.token.inserted {
+  color: #98c379;
+}
+
+.token.operator,
+.token.entity,
+.token.url,
+.language-css .token.string,
+.style .token.string {
+  color: #56b6c2;
+}
+
+.token.atrule,
+.token.keyword {
+  color: #e06c75;
+}
+
+.token.function {
+  color: #61afef;
+}
+
+.token.regex,
+.token.important,
+.token.variable {
+  color: #c678dd;
+}
+
+.token.important,
+.token.bold {
+  font-weight: bold;
+}
+
+.token.italic {
+  font-style: italic;
+}
+
+.token.entity {
+  cursor: help;
+}
+
+pre.line-numbers {
+	position: relative;
+	padding-left: 3.8em;
+	counter-reset: linenumber;
+}
+
+pre.line-numbers > code {
+	position: relative;
+}
+
+.line-numbers .line-numbers-rows {
+	position: absolute;
+	pointer-events: none;
+	top: 0;
+	font-size: 100%;
+	left: -3.8em;
+	width: 3em; /* works for line-numbers below 1000 lines */
+	letter-spacing: -1px;
+	border-right: 0;
+
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+
+}
+
+.line-numbers-rows > span {
+	pointer-events: none;
+	display: block;
+	counter-increment: linenumber;
+}
+
+.line-numbers-rows > span:before {
+	content: counter(linenumber);
+	color: #5C6370;
+	display: block;
+	padding-right: 0.8em;
+	text-align: right;
+}

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -0,0 +1,79 @@
+---
+title: How to add preparation and cleanup steps to backups
+---
+## Preparation and cleanup hooks
+
+If you find yourself performing prepraration tasks before your backup runs, or
+cleanup work afterwards, borgmatic hooks may be of interest. Hooks are shell
+commands that borgmatic executes for you at various points, and they're
+configured in the `hooks` section of your configuration file. But if you're
+looking to backup a database, it's probably easier to use the [database backup
+feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
+instead.
+
+You can specify `before_backup` hooks to perform preparation steps before
+running backups, and specify `after_backup` hooks to perform cleanup steps
+afterwards. Here's an example:
+
+```yaml
+hooks:
+    before_backup:
+        - mount /some/filesystem
+    after_backup:
+        - umount /some/filesystem
+```
+
+The `before_backup` and `after_backup` hooks each run once per configuration
+file. `before_backup` hooks run prior to backups of all repositories in a
+configuration file, right before the `create` action. `after_backup` hooks run
+afterwards, but not if an error occurs in a previous hook or in the backups
+themselves.
+
+You can also use `before_everything` and `after_everything` hooks to perform
+global setup or cleanup:
+
+```yaml
+hooks:
+    before_everything:
+        - set-up-stuff-globally
+    after_everything:
+        - clean-up-stuff-globally
+```
+
+`before_everything` hooks collected from all borgmatic configuration files run
+once before all configuration files (prior to all actions), but only if there
+is a `create` action. An error encountered during a `before_everything` hook
+causes borgmatic to exit without creating backups.
+
+`after_everything` hooks run once after all configuration files and actions,
+but only if there is a `create` action. It runs even if an error occurs during
+a backup or a backup hook, but not if an error occurs during a
+`before_everything` hook.
+
+borgmatic also runs `on_error` hooks if an error occurs, either when creating
+a backup or running a backup hook. See the [monitoring and alerting
+documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+for more information.
+
+## Hook output
+
+Any output produced by your hooks shows up both at the console and in syslog
+(when run in a non-interactive console). For more information, read about <a
+href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/">inspecting
+your backups</a>.
+
+## Security
+
+An important security note about hooks: borgmatic executes all hook commands
+with the user permissions of borgmatic itself. So to prevent potential shell
+injection or privilege escalation, do not forget to set secure permissions
+on borgmatic configuration files (`chmod 0600`) and scripts (`chmod 0700`)
+invoked by hooks.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)

docs/how-to/backup-your-databases.md

@@ -0,0 +1,181 @@
+---
+title: How to backup your databases
+---
+## Database dump hooks
+
+If you want to backup a database, it's best practice with most database
+systems to backup an exported database dump, rather than backing up your
+database's internal file storage. That's because the internal storage can
+change while you're reading from it. In contrast, a database dump creates a
+consistent snapshot that is more suited for backups.
+
+Fortunately, borgmatic includes built-in support for creating database dumps
+prior to running backups. For example, here is everything you need to dump and
+backup a couple of local PostgreSQL databases and a MySQL/MariaDB database:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+        - name: orders
+    mysql_databases:
+        - name: posts
+```
+
+Prior to each backup, borgmatic dumps each configured database to a file
+(located in `~/.borgmatic/`) and includes it in the backup. After the backup
+completes, borgmatic removes the database dump files to recover disk space.
+
+Here's a more involved example that connects to remote databases:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: users
+          hostname: database1.example.org
+          port: 5433
+          username: postgres
+          password: trustsome1
+          format: tar
+          options: "--role=someone"
+    mysql_databases:
+        - name: posts
+          hostname: database2.example.org
+          port: 3307
+          username: root
+          password: trustsome1
+          options: "--skip-comments"
+```
+
+If you want to dump all databases on a host, use `all` for the database name:
+
+```yaml
+hooks:
+    postgresql_databases:
+        - name: all
+    mysql_databases:
+        - name: all
+```
+
+Note that you may need to use a `username` of the `postgres` superuser for
+this to work with PostgreSQL.
+
+
+### Configuration backups
+
+An important note about this database configuration: You'll need the
+configuration to be present in order for borgmatic to restore a database. So
+to prepare for this situation, it's a good idea to include borgmatic's own
+configuration files as part of your regular backups. That way, you can always
+bring back any missing configuration files in order to restore a database.
+
+
+## Supported databases
+
+As of now, borgmatic supports PostgreSQL and MySQL/MariaDB databases
+directly. But see below about general-purpose preparation and cleanup hooks as
+a work-around with other database systems. Also, please [file a
+ticket](https://torsion.org/borgmatic/#issues) for additional database systems
+that you'd like supported.
+
+
+## Database restoration
+
+To restore a database dump from an archive, use the `borgmatic restore`
+action. But the first step is to figure out which archive to restore from. A
+good way to do that is to use the `list` action:
+
+```bash
+borgmatic list
+```
+
+(No borgmatic `list` action? Try the old-style `--list`, or upgrade
+borgmatic!)
+
+That should yield output looking something like:
+
+```text
+host-2019-01-01T04:05:06.070809      Tue, 2019-01-01 04:05:06 [...]
+host-2019-01-02T04:06:07.080910      Wed, 2019-01-02 04:06:07 [...]
+```
+
+Assuming that you want to restore all database dumps from the archive with the
+most up-to-date files and therefore the latest timestamp, run a command like:
+
+```bash
+borgmatic restore --archive host-2019-01-02T04:06:07.080910
+```
+
+(No borgmatic `restore` action? Upgrade borgmatic!)
+
+The `--archive` value is the name of the archive to restore from. This
+restores all databases dumps that borgmatic originally backed up to that
+archive.
+
+This is a destructive action! `borgmatic restore` replaces live databases by
+restoring dumps from the selected archive. So be very careful when and where
+you run it.
+
+
+### Repository selection
+
+If you have a single repository in your borgmatic configuration file(s), no
+problem: the `restore` action figures out which repository to use.
+
+But if you have multiple repositories configured, then you'll need to specify
+the repository path containing the archive to restore. Here's an example:
+
+```bash
+borgmatic restore --repository repo.borg --archive host-2019-...
+```
+
+### Restore particular databases
+
+If you've backed up multiple databases into an archive, and you'd only like to
+restore one of them, use the `--database` flag to select one or more
+databases. For instance:
+
+```bash
+borgmatic restore --archive host-2019-... --database users
+```
+
+### Limitations
+
+There are a few important limitations with borgmatic's current database
+restoration feature that you should know about:
+
+1. You must restore as the same Unix user that created the archive containing
+the database dump. That's because the user's home directory path is encoded
+into the path of the database dump within the archive.
+2. As mentioned above, borgmatic can only restore a database that's defined in
+borgmatic's own configuration file. So include your configuration file in
+backups to avoid getting caught without a way to restore a database.
+3. borgmatic does not currently support backing up or restoring multiple
+databases that share the exact same name on different hosts.
+
+
+### Manual restoration
+
+If you prefer to restore a database without the help of borgmatic, first
+[extract](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) an
+archive containing a database dump, and then manually restore the dump file
+found within the extracted `~/.borgmatic/` path (e.g. with `pg_restore` or
+`mysql` commands).
+
+
+## Preparation and cleanup hooks
+
+If this database integration is too limited for needs, borgmatic also supports
+general-purpose [preparation and cleanup
+hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/).
+These hooks allows you to trigger arbitrary commands or scripts before and
+after backups. So if necessary, you can use these hooks to create database
+dumps with any database system.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)

docs/how-to/deal-with-very-large-backups.md

@@ -0,0 +1,109 @@
+---
+title: How to deal with very large backups
+---
+## Biggish data
+
+Borg itself is great for efficiently de-duplicating data across successive
+backup archives, even when dealing with very large repositories. But you may
+find that while borgmatic's default mode of "prune, create, and check" works
+well on small repositories, it's not so great on larger ones. That's because
+running the default consistency checks takes a long time on large
+repositories.
+
+### A la carte actions
+
+If you find yourself in this situation, you have some options. First, you can
+run borgmatic's pruning, creating, or checking actions separately. For
+instance, the the following optional actions are available:
+
+```bash
+borgmatic prune
+borgmatic create
+borgmatic check
+```
+
+(No borgmatic `prune`, `create`, or `check` actions? Try the old-style
+`--prune`, `--create`, or `--check`. Or upgrade borgmatic!)
+
+You can run with only one of these actions provided, or you can mix and match
+any number of them in a single borgmatic run. This supports approaches like
+making backups with `create` on a frequent schedule, while only running
+expensive consistency checks with `check` on a much less frequent basis from
+a separate cron job.
+
+### Consistency check configuration
+
+Another option is to customize your consistency checks. The default
+consistency checks run both full-repository checks and per-archive checks
+within each repository.
+
+But if you find that archive checks are too slow, for example, you can
+configure borgmatic to run repository checks only. Configure this in the
+`consistency` section of borgmatic configuration:
+
+```yaml
+consistency:
+    checks:
+        - repository
+```
+
+If that's still too slow, you can disable consistency checks entirely,
+either for a single repository or for all repositories.
+
+Disabling all consistency checks looks like this:
+
+```yaml
+consistency:
+    checks:
+        - disabled
+```
+
+Or, if you have multiple repositories in your borgmatic configuration file,
+you can keep running consistency checks, but only against a subset of the
+repositories:
+
+```yaml
+consistency:
+    check_repositories:
+        - path/of/repository_to_check.borg
+```
+
+Finally, you can override your configuration file's consistency checks, and
+run particular checks via the command-line. For instance:
+
+```bash
+borgmatic check --only data --only extract
+```
+
+This is useful for running slow consistency checks on an infrequent basis,
+separate from your regular checks.
+
+
+## Troubleshooting
+
+### Broken pipe with remote repository
+
+When running borgmatic on a large remote repository, you may receive errors
+like the following, particularly while "borg check" is validating backups for
+consistency:
+
+```text
+    Write failed: Broken pipe
+    borg: Error: Connection closed by remote host
+```
+
+This error can be caused by an ssh timeout, which you can rectify by adding
+the following to the `~/.ssh/config` file on the client:
+
+```text
+    Host *
+        ServerAliveInterval 120
+```
+
+This should make the client keep the connection alive while validating
+backups.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

docs/how-to/develop-on-borgmatic.md

@@ -0,0 +1,137 @@
+---
+title: How to develop on borgmatic
+---
+## Source code
+
+To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
+
+```bash
+git clone https://projects.torsion.org/witten/borgmatic.git
+```
+
+Or:
+
+```bash
+git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git
+```
+
+Then, install borgmatic
+"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)"
+so that you can run borgmatic commands while you're hacking on them to
+make sure your changes work.
+
+```bash
+cd borgmatic/
+pip3 install --editable --user .
+```
+
+Note that this will typically install the borgmatic commands into
+`~/.local/bin`, which may or may not be on your PATH. There are other ways to
+install borgmatic editable as well, for instance into the system Python
+install (so without `--user`, as root), or even into a
+[virtualenv](https://virtualenv.pypa.io/en/stable/). How or where you install
+borgmatic is up to you, but generally an editable install makes development
+and testing easier.
+
+
+## Automated tests
+
+Assuming you've cloned the borgmatic source code as described above, and
+you're in the `borgmatic/` working copy, install tox, which is used for
+setting up testing environments:
+
+```bash
+pip3 install --user tox
+```
+
+Finally, to actually run tests, run:
+
+```bash
+cd borgmatic
+tox
+```
+
+### Code formatting
+
+If when running tests, you get an error from the
+[Black](https://black.readthedocs.io/en/stable/) code formatter about files
+that would be reformatted, you can ask Black to format them for you via the
+following:
+
+```bash
+tox -e black
+```
+
+Note that Black requires at minimum Python 3.6.
+
+And if you get a complaint from the
+[isort](https://github.com/timothycrosley/isort) Python import orderer, you
+can ask isort to order your imports for you:
+
+```bash
+tox -e isort
+```
+
+### End-to-end tests
+
+borgmatic additionally includes some end-to-end tests that integration test
+with Borg for a few representative scenarios. These tests don't run by default
+because they're relatively slow and depend on Borg. If you would like to run
+them:
+
+```bash
+tox -e end-to-end
+```
+
+## Code style
+
+Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
+the following deviations from it:
+
+ * For strings, prefer single quotes over double quotes.
+ * Limit all lines to a maximum of 100 characters.
+ * Use trailing commas within multiline values or argument lists.
+ * For multiline constructs, put opening and closing delimeters on lines
+   separate from their contents.
+ * Within multiline constructs, use standard four-space indentation. Don't align
+   indentation with an opening delimeter.
+
+borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
+formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
+the [isort](https://github.com/timothycrosley/isort) import orderer, so
+certain code style requirements will be enforced when running automated tests.
+See the Black, Flake8, and isort documentation for more information.
+
+## Continuous integration
+
+Each pull request triggers a continuous integration build which runs the test
+suite. You can view these builds on
+[build.torsion.org](https://build.torsion.org/witten/borgmatic), and they're
+also linked from the commits list on each pull request.
+
+## Documentation development
+
+Updates to borgmatic's documentation are welcome. It's formatted in Markdown
+and located in the `docs/` directory in borgmatic's source, plus the
+`README.md` file at the root.
+
+To build and view a copy of the documentation with your local changes, run the
+following from the root of borgmatic's source code:
+
+```bash
+sudo scripts/dev-docs
+```
+
+This requires Docker to be installed on your system. You may not need to use
+sudo if your non-root user has permissions to run Docker.
+
+After you run the script, you can point your web browser at
+http://localhost:8080 to view the documentation with your changes.
+
+To close the documentation server, ctrl-C the script. Note that it does not
+currently auto-reload, so you'll need to stop it and re-run it for any
+additional documentation changes to take effect.
+
+## Related documentation
+
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)

docs/how-to/extract-a-backup.md

@@ -0,0 +1,95 @@
+---
+title: How to extract a backup
+---
+## Extract
+
+When the worst happens—or you want to test your backups—the first step is
+to figure out which archive to extract. A good way to do that is to use the
+`list` action:
+
+```bash
+borgmatic list
+```
+
+(No borgmatic `list` action? Try the old-style `--list`, or upgrade
+borgmatic!)
+
+That should yield output looking something like:
+
+```text
+host-2019-01-01T04:05:06.070809      Tue, 2019-01-01 04:05:06 [...]
+host-2019-01-02T04:06:07.080910      Wed, 2019-01-02 04:06:07 [...]
+```
+
+Assuming that you want to extract the archive with the most up-to-date files
+and therefore the latest timestamp, run a command like:
+
+```bash
+borgmatic extract --archive host-2019-01-02T04:06:07.080910
+```
+
+(No borgmatic `extract` action? Try the old-style `--extract`, or upgrade
+borgmatic!)
+
+The `--archive` value is the name of the archive to extract. This extracts the
+entire contents of the archive to the current directory, so make sure you're
+in the right place before running the command.
+
+
+## Repository selection
+
+If you have a single repository in your borgmatic configuration file(s), no
+problem: the `extract` action figures out which repository to use.
+
+But if you have multiple repositories configured, then you'll need to specify
+the repository path containing the archive to extract. Here's an example:
+
+```bash
+borgmatic extract --repository repo.borg --archive host-2019-...
+```
+
+## Extract particular files
+
+Sometimes, you want to extract a single deleted file, rather than extracting
+everything from an archive. To do that, tack on one or more `--path` values.
+For instance:
+
+```bash
+borgmatic extract --archive host-2019-... --path path/1 path/2
+```
+
+Note that the specified restore paths should not have a leading slash. Like a
+whole-archive extract, this also extracts into the current directory. So for
+example, if you happen to be in the directory `/var` and you run the `extract`
+command above, borgmatic will extract `/var/path/1` and `/var/path/2`.
+
+## Extract to a particular destination
+
+By default, borgmatic extracts files into the current directory. To instead
+extract files to a particular destination directory, use the `--destination`
+flag:
+
+```bash
+borgmatic extract --archive host-2019-... --destination /tmp
+```
+
+When using the `--destination` flag, be careful not to overwrite your system's
+files with extracted files unless that is your intent.
+
+
+## Database restoration
+
+The `borgmatic extract` command only extracts files. To restore a database,
+please see the [documentation on database backups and
+restores](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/).
+borgmatic does not perform database restoration as part of `borgmatic extract`
+so that you can extract files from your archive without impacting your live
+databases.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)

docs/how-to/inspect-your-backups.md

@@ -0,0 +1,106 @@
+---
+title: How to inspect your backups
+---
+## Backup progress
+
+By default, borgmatic runs proceed silently except in the case of errors. But
+if you'd like to to get additional information about the progress of the
+backup as it proceeds, use the verbosity option:
+
+```bash
+borgmatic --verbosity 1
+```
+
+This lists the files that borgmatic is archiving, which are those that are new
+or changed since the last backup.
+
+Or, for even more progress and debug spew:
+
+```bash
+borgmatic --verbosity 2
+```
+
+## Backup summary
+
+If you're less concerned with progress during a backup, and you only want to
+see the summary of archive statistics at the end, you can use the stats
+option when performing a backup:
+
+```bash
+borgmatic --stats
+```
+
+## Existing backups
+
+borgmatic provides convenient actions for Borg's
+[list](https://borgbackup.readthedocs.io/en/stable/usage/list.html) and
+[info](https://borgbackup.readthedocs.io/en/stable/usage/info.html)
+functionality:
+
+
+```bash
+borgmatic list
+borgmatic info
+```
+
+(No borgmatic `list` or `info` actions? Try the old-style `--list` or
+`--info`. Or upgrade borgmatic!)
+
+
+## Logging
+
+By default, borgmatic logs to a local syslog-compatible daemon if one is
+present and borgmatic is running in a non-interactive console. Where those
+logs show up depends on your particular system. If you're using systemd, try
+running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or
+similiar.
+
+You can customize the log level used for syslog logging with the
+`--syslog-verbosity` flag, and this is independent from the console logging
+`--verbosity` flag described above. For instance, to get additional
+information about the progress of the backup as it proceeds:
+
+```bash
+borgmatic --syslog-verbosity 1
+```
+
+Or to increase syslog logging to include debug spew:
+
+```bash
+borgmatic --syslog-verbosity 2
+```
+
+### Logging to file
+
+If you don't want to use syslog, and you'd rather borgmatic log to a plain
+file, use the `--log-file` flag:
+
+```bash
+borgmatic --log-file /path/to/file.log
+```
+
+Note that if you use the `--log-file` flag, you are responsible for rotating
+the log file so it doesn't grow too large. Also, there is a
+`--log-file-verbosity` flag to customize the log file's log level.
+
+
+### systemd journal
+
+If your local syslog daemon is systemd's journal, be aware that journald by
+default throttles the rate at which a particular program can log. So you may
+need to [change the journald rate
+limit](https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=)
+in `/etc/systemd/journald.conf` if you're finding that borgmatic journald logs
+are missing.
+
+Note that the [sample borgmatic systemd service
+file](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#systemd)
+already has this rate limit disabled.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)

docs/how-to/make-per-application-backups.md

@@ -0,0 +1,115 @@
+---
+title: How to make per-application backups
+---
+## Multiple backup configurations
+
+You may find yourself wanting to create different backup policies for
+different applications on your system. For instance, you may want one backup
+configuration for your database data directory, and a different configuration
+for your user home directories.
+
+The way to accomplish that is pretty simple: Create multiple separate
+configuration files and place each one in a `/etc/borgmatic.d/` directory. For
+instance:
+
+```bash
+sudo mkdir /etc/borgmatic.d
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+```
+
+When you set up multiple configuration files like this, borgmatic will run
+each one in turn from a single borgmatic invocation. This includes, by
+default, the traditional `/etc/borgmatic/config.yaml` as well.
+
+And if you need even more customizability, you can specify alternate
+configuration paths on the command-line with borgmatic's `--config` option.
+See `borgmatic --help` for more information.
+
+
+## Configuration includes
+
+Once you have multiple different configuration files, you might want to share
+common configuration options across these files with having to copy and paste
+them. To achieve this, you can put fragments of common configuration options
+into a file, and then include or inline that file into one or more borgmatic
+configuration files.
+
+Let's say that you want to include common retention configuration across all
+of your configuration files. You could do that in each configuration file with
+the following:
+
+```yaml
+location:
+   ...
+
+retention:
+    !include /etc/borgmatic/common_retention.yaml
+```
+
+And then the contents of `common_retention.yaml` could be:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+To prevent borgmatic from trying to load these configuration fragments by
+themselves and complaining that they are not valid configuration files, you
+should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
+is fine.)
+
+Note that this form of include must be a YAML value rather than a key. For
+example, this will not work:
+
+```yaml
+location:
+   ...
+
+# Don't do this. It won't work!
+!include /etc/borgmatic/common_retention.yaml
+```
+
+But if you do want to merge in a YAML key and its values, keep reading!
+
+
+## Include merging
+
+If you need to get even fancier and pull in common configuration options while
+potentially overriding individual options, you can perform a YAML merge of
+included configuration using the YAML `<<` key. For instance, here's an
+example of a main configuration file that pulls in two retention options via
+an include, and then overrides one of them locally:
+
+```yaml
+location:
+   ...
+
+retention:
+    keep_daily: 5
+    <<: !include /etc/borgmatic/common_retention.yaml
+```
+
+This is what `common_retention.yaml` might look like:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration would have a
+`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
+
+When there is a collision of an option between the local file and the merged
+include, the local file's option takes precedent. And note that this is a
+shallow merge rather than a deep merge, so the merging does not descend into
+nested values.
+
+Note that this `<<` include merging syntax is only for merging in mappings
+(keys/values). If you'd like to include other types like scalars or lists
+directly, please see the section above about standard includes.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

docs/how-to/monitor-your-backups.md

@@ -0,0 +1,226 @@
+---
+title: How to monitor your backups
+---
+
+## Monitoring and alerting
+
+Having backups is great, but they won't do you a lot of good unless you have
+confidence that they're running on a regular basis. That's where monitoring
+and alerting comes in.
+
+There are several different ways you can monitor your backups and find out
+whether they're succeeding. Which of these you choose to do is up to you and
+your particular infrastructure:
+
+1. **Job runner alerts**: The easiest place to start is with failure alerts
+from the [scheduled job
+runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) (cron,
+systemd, etc.) that's running borgmatic. But note that if the job doesn't even
+get scheduled (e.g. due to the job runner not running), you probably won't get
+an alert at all! Still, this is a decent first line of defense, especially
+when combined with some of the other approaches below.
+2. **borgmatic error hooks**: The `on_error` hook allows you to run an arbitrary
+command or script when borgmatic itself encounters an error running your
+backups. So for instance, you can run a script to send yourself a text message
+alert. But note that if borgmatic doesn't actually run, this alert won't fire.
+See [error
+hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
+below for how to configure this.
+4. **borgmatic monitoring hooks**: This feature integrates with monitoring
+   services like [Healthchecks](https://healthchecks.io/),
+[Cronitor](https://cronitor.io), and [Cronhub](https://cronhub.io), and pings
+these services whenever borgmatic runs. That way, you'll receive an alert when
+something goes wrong or the service doesn't hear from borgmatic for a
+configured interval. See
+[Healthchecks
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), and [Cronhub
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook)
+below for how to configure this.
+3. **Third-party monitoring software**: You can use traditional monitoring
+software to consume borgmatic JSON output and track when the last
+successful backup occurred. See [scripting
+borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
+below for how to configure this.
+5. **Borg hosting providers**: Most [Borg hosting
+providers](https://torsion.org/borgmatic/#hosting-providers) include
+monitoring and alerting as part of their offering. This gives you a dashboard
+to check on all of your backups, and can alert you if the service doesn't hear
+from borgmatic for a configured interval.
+6. **borgmatic consistency checks**: While not strictly part of monitoring, if you
+really want confidence that your backups are not only running but are
+restorable as well, you can configure particular [consistency
+checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration)
+or even script full [extract
+tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/).
+
+
+## Error hooks
+
+When an error occurs during a backup, borgmatic can run configurable shell
+commands to fire off custom error notifications or take other actions, so you
+can get alerted as soon as something goes wrong. Here's a not-so-useful
+example:
+
+```yaml
+hooks:
+    on_error:
+        - echo "Error while creating a backup or running a backup hook."
+```
+
+The `on_error` hook supports interpolating particular runtime variables into
+the hook command. Here's an example that assumes you provide a separate shell
+script to handle the alerting:
+
+```yaml
+hooks:
+    on_error:
+        - send-text-message.sh "{configuration_filename}" "{repository}"
+```
+
+In this example, when the error occurs, borgmatic interpolates a few runtime
+values into the hook command: the borgmatic configuration filename, and the
+path of the repository. Here's the full set of supported variables you can use
+here:
+
+ * `configuration_filename`: borgmatic configuration filename in which the
+   error occurred
+ * `repository`: path of the repository in which the error occurred (may be
+   blank if the error occurs in a hook)
+ * `error`: the error message itself
+ * `output`: output of the command that failed (may be blank if an error
+   occurred without running a command)
+
+Note that borgmatic does not run `on_error` hooks if an error occurs within a
+`before_everything` or `after_everything` hook. For more about hooks, see the
+[borgmatic hooks
+documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
+especially the security information.
+
+
+## Healthchecks hook
+
+[Healthchecks](https://healthchecks.io/) is a service that provides "instant
+alerts when your cron jobs fail silently", and borgmatic has built-in
+integration with it. Once you create a Healthchecks account and project on
+their site, all you need to do is configure borgmatic with the unique "Ping
+URL" for your project. Here's an example:
+
+
+```yaml
+hooks:
+    healthchecks: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
+```
+
+With this hook in place, borgmatic pings your Healthchecks project when a
+backup begins, ends, or errors. Specifically, before the <a
+href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
+hooks</a> run, borgmatic lets Healthchecks know that a backup has started.
+Then, if the backup completes successfully, borgmatic notifies Healthchecks of
+the success after the `after_backup` hooks run. And if an error occurs during
+the backup, borgmatic notifies Healthchecks after the `on_error` hooks run.
+
+You can configure Healthchecks to notify you by a [variety of
+mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
+or it doesn't hear from borgmatic for a certain period of time.
+
+
+## Cronitor hook
+
+[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
+for websites, services and APIs", and borgmatic has built-in
+integration with it. Once you create a Cronitor account and cron job monitor on
+their site, all you need to do is configure borgmatic with the unique "Ping
+API URL" for your monitor. Here's an example:
+
+
+```yaml
+hooks:
+    cronitor: https://cronitor.link/d3x0c1
+```
+
+With this hook in place, borgmatic pings your Cronitor monitor when a backup
+begins, ends, or errors. Specifically, before the <a
+href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
+hooks</a> run, borgmatic lets Cronitor know that a backup has started. Then,
+if the backup completes successfully, borgmatic notifies Cronitor of the
+success after the `after_backup` hooks run. And if an error occurs during the
+backup, borgmatic notifies Cronitor after the `on_error` hooks run.
+
+You can configure Cronitor to notify you by a [variety of
+mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
+or it doesn't hear from borgmatic for a certain period of time.
+
+
+## Cronhub hook
+
+[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
+background jobs fail silently or run longer than expected", and borgmatic has
+built-in integration with it. Once you create a Cronhub account and monitor on
+their site, all you need to do is configure borgmatic with the unique "Ping
+URL" for your monitor. Here's an example:
+
+
+```yaml
+hooks:
+    cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
+```
+
+With this hook in place, borgmatic pings your Cronhub monitor when a backup
+begins, ends, or errors. Specifically, before the <a
+href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
+hooks</a> run, borgmatic lets Cronhub know that a backup has started. Then,
+if the backup completes successfully, borgmatic notifies Cronhub of the
+success after the `after_backup` hooks run. And if an error occurs during the
+backup, borgmatic notifies Cronhub after the `on_error` hooks run.
+
+Note that even though you configure borgmatic with the "start" variant of the
+ping URL, borgmatic substitutes the correct state into the URL when pinging
+Cronhub ("start", "finish", or "fail").
+
+You can configure Cronhub to notify you by a [variety of
+mechanisms](https://docs.cronhub.io/integrations.html) when backups fail
+or it doesn't hear from borgmatic for a certain period of time.
+
+
+## Scripting borgmatic
+
+To consume the output of borgmatic in other software, you can include an
+optional `--json` flag with `create`, `list`, or `info` to get the output
+formatted as JSON.
+
+Note that when you specify the `--json` flag, Borg's other non-JSON output is
+suppressed so as not to interfere with the captured JSON. Also note that JSON
+output only shows up at the console, and not in syslog.
+
+
+### Successful backups
+
+`borgmatic list` includes support for a `--successful` flag that only lists
+successful (non-checkpoint) backups. This flag works via a basic heuristic: It
+assumes that non-checkpoint archive names end with a digit (e.g. from a
+timestamp), while checkpoint archive names do not. This means that if you're
+using custom archive names that do not end in a digit, the `--successful` flag
+will not work as expected.
+
+Combined with a built-in Borg flag like `--last`, you can list the last
+successful backup for use in your monitoring scripts. Here's an example
+combined with `--json`:
+
+```bash
+borgmatic list --successful --last 1 --json
+```
+
+Note that this particular combination will only work if you've got a single
+backup "series" in your repository. If you're instead backing up, say, from
+multiple different hosts into a single repository, then you'll need to get
+fancier with your archive listing. See `borg list --help` for more flags.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)

docs/how-to/restore-a-backup.md

@@ -0,0 +1,3 @@
+<head>
+    <meta http-equiv='refresh' content='0; URL=https://torsion.org/borgmatic/docs/how-to/extract-a-backup/'>
+</head>

docs/how-to/run-preparation-steps-before-backups.md

@@ -0,0 +1,3 @@
+<head>
+    <meta http-equiv='refresh' content='0; URL=https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/'>
+</head>

docs/how-to/set-up-backups.md

@@ -0,0 +1,262 @@
+---
+title: How to set up backups with borgmatic
+---
+## Installation
+
+To get up and running, first [install
+Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at
+least version 1.1.
+
+By default, borgmatic looks for its configuration files in `/etc/borgmatic/`
+and `/etc/borgmatic.d/`, where the root user typically has read access.
+
+So, to download and install borgmatic as the root user, run the following
+commands:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+This is a [recommended user site
+installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site).
+You will need to ensure that `/root/.local/bin` is available on your `$PATH`
+so
+that the borgmatic executable is available. For instance, adding this to
+root's `~/.profile` or `~/.bash_profile` may do the trick:
+
+```bash
+export PATH="$PATH:~/.local/bin"
+```
+
+Note that your pip binary may have a different name than "pip3". Make sure
+you're using Python 3, as borgmatic does not support Python 2.
+
+### Other ways to install
+
+Along with the above process, you have several other options for installing
+borgmatic:
+
+ * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
+ * [Docker image with support for scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
+ * [Debian](https://tracker.debian.org/pkg/borgmatic)
+ * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
+ * [Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
+ * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
+ * [OpenBSD](http://ports.su/sysutils/borgmatic)
+ * [openSUSE](https://software.opensuse.org/package/borgmatic)
+ * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
+ * [virtualenv](https://virtualenv.pypa.io/en/stable/)
+
+
+## Hosting providers
+
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
+
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
+
+## Configuration
+
+After you install borgmatic, generate a sample configuration file:
+
+```bash
+sudo generate-borgmatic-config
+```
+
+If that command is not found, then it may be installed in a location that's
+not in your system `PATH` (see above). Try looking in `~/.local/bin/`.
+
+This generates a sample configuration file at /etc/borgmatic/config.yaml (by
+default). You should edit the file to suit your needs, as the values are
+representative. All options are optional except where indicated, so feel free
+to ignore anything you don't need.
+
+Note that the configuration file is organized into distinct sections, each
+with a section name like `location:` or `storage:`. So take care that if you
+uncomment a particular option, also uncomment its containing section name, or
+else borgmatic won't recognize the option. Also be sure to use spaces rather
+than tabs for indentation; YAML does not allow tabs.
+
+You can also get the same sample configuration file from the [configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration/), the
+authoritative set of all configuration options. This is handy if borgmatic has
+added new options
+since you originally created your configuration file. Also check out how to
+[upgrade your
+configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration).
+
+
+### Encryption
+
+Note that if you plan to run borgmatic on a schedule with cron, and you
+encrypt your Borg repository with a passphrase instead of a key file, you'll
+either need to set the borgmatic `encryption_passphrase` configuration
+variable or set the `BORG_PASSPHRASE` environment variable. See the
+[repository encryption
+section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)
+of the Borg Quick Start for more info.
+
+Alternatively, you can specify the passphrase programatically by setting
+either the borgmatic `encryption_passcommand` configuration variable or the
+`BORG_PASSCOMMAND` environment variable. See the [Borg Security
+FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
+for more info.
+
+
+### Validation
+
+If you'd like to validate that your borgmatic configuration is valid, the
+following command is available for that:
+
+```bash
+sudo validate-borgmatic-config
+```
+
+This command's exit status (`$?` in Bash) is zero when configuration is valid
+and non-zero otherwise.
+
+Validating configuration can be useful if you generate your configuration
+files via configuration management, or you want to double check that your hand
+edits are valid.
+
+
+## Initialization
+
+Before you can create backups with borgmatic, you first need to initialize a
+Borg repository so you have a destination for your backup archives. (But skip
+this step if you already have a Borg repository.) To create a repository, run
+a command like the following:
+
+```bash
+borgmatic init --encryption repokey
+```
+
+(No borgmatic `init` action? Try the old-style `--init` flag, or upgrade
+borgmatic!)
+
+This uses the borgmatic configuration file you created above to determine
+which local or remote repository to create, and encrypts it with the
+encryption passphrase specified there if one is provided. Read about [Borg
+encryption
+modes](https://borgbackup.readthedocs.io/en/stable/usage/init.html#encryption-modes)
+for the menu of available encryption modes.
+
+Also, optionally check out the [Borg Quick
+Start](https://borgbackup.readthedocs.org/en/stable/quickstart.html) for more
+background about repository initialization.
+
+Note that borgmatic skips repository initialization if the repository already
+exists. This supports use cases like ensuring a repository exists prior to
+performing a backup.
+
+If the repository is on a remote host, make sure that your local user has
+key-based SSH access to the desired user account on the remote host.
+
+
+## Backups
+
+Now that you've configured borgmatic and initialized a repository, it's a
+good idea to test that borgmatic is working. So to run borgmatic and start a
+backup, you can invoke it like this:
+
+```bash
+borgmatic --verbosity 1
+```
+
+By default, this will also prune any old backups as per the configured
+retention policy, and check backups for consistency problems due to things
+like file damage.
+
+The verbosity flag makes borgmatic list the files that it's archiving, which
+are those that are new or changed since the last backup. Eyeball the list and
+see if it matches your expectations based on the configuration.
+
+
+## Autopilot
+
+Running backups manually is good for validating your configuration, but I'm
+guessing that you want to run borgmatic automatically, say once a day. To do
+that, you can configure a separate job runner to invoke it periodically.
+
+### cron
+
+If you're using cron, download the [sample cron
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic).
+Then, from the directory where you downloaded it:
+
+```bash
+sudo mv borgmatic /etc/cron.d/borgmatic
+sudo chmod +x /etc/cron.d/borgmatic
+```
+
+You can modify the cron file if you'd like to run borgmatic more or less frequently.
+
+### systemd
+
+If you're using systemd instead of cron to run jobs, download the [sample
+systemd service
+file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
+and the [sample systemd timer
+file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
+Then, from the directory where you downloaded them:
+
+```bash
+sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
+sudo systemctl enable borgmatic.timer
+sudo systemctl start borgmatic.timer
+```
+
+Feel free to modify the timer file based on how frequently you'd like
+borgmatic to run.
+
+## Colored output
+
+Borgmatic produces colored terminal output by default. It is disabled when a
+non-interactive terminal is detected (like a cron job). Otherwise, you can
+disable it by passing the `--no-color` flag, setting the environment variable
+`PY_COLORS=False`, or setting the `color` option to `false` in the `output`
+section of configuration.
+
+## Troubleshooting
+
+### "found character that cannot start any token" error
+
+If you run borgmatic and see an error looking something like this, it probably
+means you've used tabs instead of spaces:
+
+```
+test.yaml: Error parsing configuration file
+An error occurred while parsing a configuration file at config.yaml:
+while scanning for the next token
+found character that cannot start any token
+  in "config.yaml", line 230, column 1
+```
+
+YAML does not allow tabs. So to fix this, replace any tabs in your
+configuration file with the requisite number of spaces.
+
+### libyaml compilation errors
+
+borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
+use a C YAML library (libyaml) if present. But if it's not installed, then
+when installing or upgrading borgmatic, you may see errors about compiling the
+YAML library. If so, not to worry. borgmatic should install and function
+correctly even without the C YAML library. And borgmatic won't be any faster
+with the C library present, so you don't need to go out of your way to install
+it.
+
+
+## Related documentation
+
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)

docs/how-to/upgrade.md

@@ -0,0 +1,118 @@
+---
+title: How to upgrade borgmatic
+---
+## Upgrading
+
+In general, all you should need to do to upgrade borgmatic is run the
+following:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+See below about special cases with old versions of borgmatic. Additionally, if
+you installed borgmatic [without using `pip3 install
+--user`](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install),
+then your upgrade process may be different.
+
+
+### Upgrading your configuration
+
+The borgmatic configuration file format is almost always backwards-compatible
+from release to release without any changes, but you may still want to update
+your configuration file when you upgrade to take advantage of new
+configuration options. This is completely optional. If you prefer, you can add
+new configuration options manually.
+
+If you do want to upgrade your configuration file to include new options, use
+the `generate-borgmatic-config` script with its optional `--source` flag that
+takes the path to your original configuration file. If provided with this
+path, `generate-borgmatic-config` merges your original configuration into the
+generated configuration file, so you get all the newest options and comments.
+
+Here's an example:
+
+```bash
+generate-borgmatic-config --source config.yaml --destination config-new.yaml
+```
+
+New options start as commented out, so you can edit the file and decide
+whether you want to use each one.
+
+There are a few caveats to this process, however. First, when generating the
+new configuration file, `generate-borgmatic-config` replaces any comments
+you've written in your original configuration file with the newest generated
+comments. Second, the script adds back any options you had originally deleted,
+although it does so with the options commented out. And finally, any YAML
+includes you've used in the source configuration get flattened out into a
+single generated file.
+
+As a safety measure, `generate-borgmatic-config` refuses to modify
+configuration files in-place. So it's up to you to review the generated file
+and, if desired, replace your original configuration file with it.
+
+
+### Upgrading from borgmatic 1.0.x
+
+borgmatic changed its configuration file format in version 1.1.0 from
+INI-style to YAML. This better supports validation, and has a more natural way
+to express lists of values. To upgrade your existing configuration, first
+upgrade to the new version of borgmatic.
+
+As of version 1.1.0, borgmatic no longer supports Python 2. If you were
+already running borgmatic with Python 3, then you can upgrade borgmatic
+in-place:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+But if you were running borgmatic with Python 2, uninstall and reinstall instead:
+
+```bash
+sudo pip uninstall borgmatic
+sudo pip3 install --user borgmatic
+```
+
+The pip binary names for different versions of Python can differ, so the above
+commands may need some tweaking to work on your machine.
+
+
+Once borgmatic is upgraded, run:
+
+```bash
+sudo upgrade-borgmatic-config
+```
+
+That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
+(by default) using the values from both your existing configuration and
+excludes files. The new version of borgmatic will consume the YAML
+configuration file instead of the old one.
+
+
+### Upgrading from atticmatic
+
+You can ignore this section if you're not an atticmatic user (the former name
+of borgmatic).
+
+borgmatic only supports Borg now and no longer supports Attic. So if you're
+an Attic user, consider switching to Borg. See the [Borg upgrade
+command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
+for more information. Then, follow the instructions above about setting up
+your borgmatic configuration files.
+
+If you were already using Borg with atticmatic, then you can upgrade
+from atticmatic to borgmatic by running the following commands:
+
+```bash
+sudo pip3 uninstall atticmatic
+sudo pip3 install --user borgmatic
+```
+
+That's it! borgmatic will continue using your /etc/borgmatic configuration
+files.
+
+
+## Related documentation
+
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)

docs/reference/command-line.md

@@ -0,0 +1,17 @@
+---
+title: borgmatic command-line reference
+---
+## borgmatic options
+
+Here are all of the available borgmatic command-line options. This includes the separate options for
+each action sub-command:
+
+```
+{% include borgmatic/command-line.txt %}
+```
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)

docs/reference/configuration.md

@@ -0,0 +1,19 @@
+---
+title: borgmatic configuration reference
+---
+## Configuration file
+
+Here is a full sample borgmatic configuration file including all available options: 
+
+```yaml
+{% include borgmatic/config.yaml %}
+```
+
+Note that you can also [download this configuration
+file](https://torsion.org/borgmatic/docs/reference/config.yaml) for use locally.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 100
+skip-string-normalization = true

sample/cron-alpine/borgmatic

@@ -1,3 +0,0 @@
-# You can drop this file into /etc/cron.d/ to run borgmatic nightly.
-
-0 3 * * * PATH=$PATH:/usr/bin /usr/bin/borgmatic

sample/cron/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/borgmatic
+0 3 * * * root PATH=$PATH:/usr/bin:/usr/local/bin /root/.local/bin/borgmatic --syslog-verbosity 1

sample/systemd/borgmatic.service

@@ -1,6 +1,22 @@
 [Unit]
 Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+ConditionACPower=true
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/borgmatic
+
+# Lower CPU and I/O priority.
+Nice=19
+CPUSchedulingPolicy=batch
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+IOWeight=100
+
+Restart=no
+LogRateLimitIntervalSec=0
+
+# Delay start to prevent backups running during boot.
+ExecStartPre=sleep 1m
+ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --syslog-verbosity 1

scripts/dev-docs

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile .
+echo
+echo "You can view dev docs at http://localhost:8080"
+echo
+docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs

scripts/find-unsupported-borg-options

@@ -18,7 +18,7 @@ mv temp.yaml.uncommented temp.yaml
 for sub_command in prune create check list info; do
     echo "********** borg $sub_command **********"
 
-    for line in $(borgmatic --config temp.yaml --$sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
+    for line in $(borgmatic --config temp.yaml $sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
         echo "$line" | grep '^-' >> borgmatic_borg_flags
     done
     sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
@@ -40,12 +40,25 @@ for sub_command in prune create check list info; do
             | grep -v '^--list$' \
             | grep -v '^--nobsdflags$' \
             | grep -v '^--pattern$' \
+            | grep -v '^--progress$' \
+            | grep -v '^--stats$' \
             | grep -v '^--read-special$' \
             | grep -v '^--repository-only$' \
             | grep -v '^--show-rc$' \
             | grep -v '^--stats$' \
             | grep -v '^--verbose$' \
             | grep -v '^--warning$' \
+            | grep -v '^--exclude' \
+            | grep -v '^--exclude-from' \
+            | grep -v '^--first' \
+            | grep -v '^--format' \
+            | grep -v '^--glob-archives' \
+            | grep -v '^--last' \
+            | grep -v '^--list-format' \
+            | grep -v '^--patterns-from' \
+            | grep -v '^--prefix' \
+            | grep -v '^--short' \
+            | grep -v '^--sort-by' \
             | grep -v '^-h$' \
             >> all_borg_flags
     done

scripts/push

@@ -2,5 +2,7 @@
 
 set -e
 
-git push -u origin master
-git push -u github master
+branch_name=$(git rev-parse --abbrev-ref HEAD)
+
+git push -u github "$branch_name"
+git push -u origin "$branch_name"

scripts/release

@@ -27,9 +27,9 @@ twine upload -r pypi dist/borgmatic-*.tar.gz
 twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
 
 # Set release changelogs on projects.evoworx.org and GitHub.
-release_changelog=$(sed '/^$/q' NEWS |grep '^\s*\*')
-escaped_release_changelog=$(echo $release_changelog | sed -z 's/\n/\\n/g')
-curl --request POST \
+release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
+escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
+curl --silent --request POST \
     "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \
     --header "Accept: application/json" \
     --header "Content-Type: application/json" \

scripts/run-tests

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# This script is intended to be run from the continuous integration build
+# server, and not on a developer machine. For that, see:
+# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
+
+set -e
+
+python -m pip install --upgrade pip==19.3.1
+pip install tox==3.14.0
+tox
+apk add --no-cache borgbackup
+tox -e end-to-end

setup.cfg

@@ -1,2 +1,20 @@
 [metadata]
 description-file=README.md
+
+[tool:pytest]
+testpaths = tests
+addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end
+filterwarnings = 
+    ignore:Coverage disabled.*:pytest.PytestWarning
+
+[flake8]
+ignore = E501,W503
+exclude = *.*/*
+
+[tool:isort]
+force_single_line = False
+include_trailing_comma = True
+known_first_party = borgmatic
+line_length = 100
+multi_line_output = 3
+skip = .tox

setup.py

@@ -1,17 +1,16 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-
-VERSION = '1.2.8'
+VERSION = '1.4.9'
 
 
 setup(
     name='borgmatic',
     version=VERSION,
-    description='A wrapper script for Borg backup software that creates and prunes backups',
+    description='Simple, configuration-driven backup software for servers and workstations',
     author='Dan Helfman',
     author_email='witten@torsion.org',
     url='https://torsion.org/borgmatic',
-    classifiers=(
+    classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
         'Intended Audience :: System Administrators',
@@ -19,17 +18,23 @@ setup(
         'Programming Language :: Python',
         'Topic :: Security :: Cryptography',
         'Topic :: System :: Archiving :: Backup',
-    ),
-    packages=find_packages(),
+    ],
+    packages=find_packages(exclude=['tests*']),
     entry_points={
         'console_scripts': [
             'borgmatic = borgmatic.commands.borgmatic:main',
             'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
             'generate-borgmatic-config = borgmatic.commands.generate_config:main',
+            'validate-borgmatic-config = borgmatic.commands.validate_config:main',
         ]
     },
     obsoletes=['atticmatic'],
-    install_requires=('pykwalify>=1.6.0,<14.06', 'ruamel.yaml>0.15.0,<0.16.0', 'setuptools'),
-    tests_require=('flexmock', 'pytest'),
+    install_requires=(
+        'pykwalify>=1.6.0,<14.06',
+        'requests',
+        'ruamel.yaml>0.15.0,<0.17.0',
+        'setuptools',
+        'colorama>=0.4.1,<0.5',
+    ),
     include_package_data=True,
 )

test_requirements.txt

@@ -1,23 +1,25 @@
 appdirs==1.4.3
-atomicwrites==1.2.1
-attrs==18.2.0
-black==18.9b0
-Click==7.0
-coverage==4.5.1
+atomicwrites==1.3.0
+attrs==19.3.0
+black==19.3b0; python_version >= '3.6'
+click==7.0
+colorama==0.4.1
+coverage==4.5.4
 docopt==0.6.2
-flake8==3.5.0
-flexmock==0.10.2
+flake8==3.7.9
+flexmock==0.10.4
+isort==4.3.21
 mccabe==0.6.1
-more-itertools==4.3.0
-pluggy==0.7.1
-py==1.6.0
-pycodestyle==2.3.1
-pyflakes==2.0.0
+more-itertools==7.2.0
+pluggy==0.13.0
+py==1.8.0
+pycodestyle==2.5.0
+pyflakes==2.1.1
 pykwalify==1.7.0
-pytest==3.8.2
-pytest-cov==2.6.0
-python-dateutil==2.7.3
-PyYAML==3.13
-ruamel.yaml>0.15.0,<0.16.0
-six==1.11.0
+pytest==5.2.2
+pytest-cov==2.8.1
+python-dateutil==2.8.0
+PyYAML==5.1.2
+requests==2.22.0
+ruamel.yaml>0.15.0,<0.17.0
 toml==0.10.0

tests/end-to-end/test_borgmatic.py

@@ -9,16 +9,20 @@ import tempfile
 def generate_configuration(config_path, repository_path):
     '''
     Generate borgmatic configuration into a file at the config path, and update the defaults so as
-    to work for testing (including injecting the given repository path).
+    to work for testing (including injecting the given repository path and tacking on an encryption
+    passphrase).
     '''
-    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+    subprocess.check_call(
+        'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+    )
     config = (
         open(config_path)
         .read()
         .replace('user@backupserver:sourcehostname.borg', repository_path)
-        .replace('- /home', f'- {config_path}')
+        .replace('- /home', '- {}'.format(config_path))
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')
+        + 'storage:\n    encryption_passphrase: "test"'
     )
     config_file = open(config_path, 'w')
     config_file.write(config)
@@ -29,35 +33,49 @@ def test_borgmatic_command():
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()
     repository_path = os.path.join(temporary_directory, 'test.borg')
+    extract_path = os.path.join(temporary_directory, 'extract')
+
+    original_working_directory = os.getcwd()
+    os.mkdir(extract_path)
+    os.chdir(extract_path)
 
     try:
-        subprocess.check_call(
-            f'borg init --encryption repokey {repository_path}'.split(' '),
-            env={'BORG_PASSPHRASE': '', **os.environ},
-        )
-
         config_path = os.path.join(temporary_directory, 'test.yaml')
         generate_configuration(config_path, repository_path)
 
-        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
-        subprocess.check_call(f'borgmatic --config {config_path}'.split(' '))
-        output = subprocess.check_output(
-            f'borgmatic --config {config_path} --list --json'.split(' '),
-            encoding=sys.stdout.encoding,
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} --init --encryption repokey'.format(config_path).split(' ')
         )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call('borgmatic --config {}'.format(config_path).split(' '))
+        output = subprocess.check_output(
+            'borgmatic --config {} --list --json'.format(config_path).split(' ')
+        ).decode(sys.stdout.encoding)
         parsed_output = json.loads(output)
 
         assert len(parsed_output) == 1
         assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
 
-        # Also exercise the info flag.
+        # Extract the created archive into the current (temporary) directory, and confirm that the
+        # extracted file looks right.
         output = subprocess.check_output(
-            f'borgmatic --config {config_path} --info --json'.split(' '),
-            encoding=sys.stdout.encoding,
-        )
+            'borgmatic --config {} --extract --archive {}'.format(config_path, archive_name).split(
+                ' '
+            )
+        ).decode(sys.stdout.encoding)
+        extracted_config_path = os.path.join(extract_path, config_path)
+        assert open(extracted_config_path).read() == open(config_path).read()
+
+        # Exercise the info flag.
+        output = subprocess.check_output(
+            'borgmatic --config {} --info --json'.format(config_path).split(' ')
+        ).decode(sys.stdout.encoding)
         parsed_output = json.loads(output)
 
         assert len(parsed_output) == 1
         assert 'repository' in parsed_output[0]
     finally:
+        os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)

tests/end-to-end/test_validate_config.py

@@ -0,0 +1,36 @@
+import os
+import subprocess
+import tempfile
+
+
+def test_validate_config_command_with_valid_configuration_succeeds():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 0
+
+
+def test_validate_config_command_with_invalid_configuration_fails():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"')
+        config_file = open(config_path, 'w')
+        config_file.write(config)
+        config_file.close()
+
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 1

tests/integration/commands/test_arguments.py

@@ -0,0 +1,423 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.commands import arguments as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments()
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == 0
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == ['myconfig', 'otherconfig']
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == 0
+
+
+def test_parse_arguments_with_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--verbosity', '1')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 1
+    assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == 0
+
+
+def test_parse_arguments_with_syslog_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--syslog-verbosity', '2')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 2
+
+
+def test_parse_arguments_with_log_file_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--log-file-verbosity', '-1')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == -1
+
+
+def test_parse_arguments_with_list_json_overrides_default():
+    arguments = module.parse_arguments('list', '--json')
+
+    assert 'list' in arguments
+    assert arguments['list'].json is True
+
+
+def test_parse_arguments_with_dashed_list_json_overrides_default():
+    arguments = module.parse_arguments('--list', '--json')
+
+    assert 'list' in arguments
+    assert arguments['list'].json is True
+
+
+def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments()
+
+    assert 'prune' in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--stats')
+
+    assert 'prune' in arguments
+    assert arguments['prune'].stats
+    assert 'create' in arguments
+    assert arguments['create'].stats
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_help_and_no_actions_shows_global_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'global arguments:' in captured.out
+    assert 'actions:' in captured.out
+
+
+def test_parse_arguments_with_help_and_action_shows_action_help(capsys):
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit) as exit:
+        module.parse_arguments('create', '--help')
+
+    assert exit.value.code == 0
+    captured = capsys.readouterr()
+    assert 'global arguments:' not in captured.out
+    assert 'actions:' not in captured.out
+    assert 'create arguments:' in captured.out
+
+
+def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('prune')
+
+    assert 'prune' in arguments
+    assert 'create' not in arguments
+    assert 'check' not in arguments
+
+
+def test_parse_arguments_with_dashed_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--prune')
+
+    assert 'prune' in arguments
+    assert 'create' not in arguments
+    assert 'check' not in arguments
+
+
+def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('create', 'check')
+
+    assert 'prune' not in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_multiple_dashed_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--create', '--check')
+
+    assert 'prune' not in arguments
+    assert 'create' in arguments
+    assert 'check' in arguments
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')
+
+
+def test_parse_arguments_disallows_deprecated_excludes_option():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+
+def test_parse_arguments_disallows_encryption_mode_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
+
+
+def test_parse_arguments_allows_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey')
+
+
+def test_parse_arguments_allows_encryption_mode_with_dashed_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey')
+
+
+def test_parse_arguments_requires_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', 'init')
+
+
+def test_parse_arguments_disallows_append_only_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--append-only')
+
+
+def test_parse_arguments_disallows_storage_quota_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
+
+
+def test_parse_arguments_allows_init_and_prune():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune')
+
+
+def test_parse_arguments_allows_init_and_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create')
+
+
+def test_parse_arguments_disallows_init_and_dry_run():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments(
+            '--config', 'myconfig', 'init', '--encryption', 'repokey', '--dry-run'
+        )
+
+
+def test_parse_arguments_disallows_glob_archives_with_successful():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments(
+            '--config', 'myconfig', 'list', '--glob-archives', '*glob*', '--successful'
+        )
+
+
+def test_parse_arguments_disallows_repository_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
+
+
+def test_parse_arguments_allows_repository_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments(
+        '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test'
+    )
+
+
+def test_parse_arguments_allows_repository_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg')
+
+
+def test_parse_arguments_disallows_archive_without_extract_restore_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--archive', 'test')
+
+
+def test_parse_arguments_disallows_paths_without_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', '--path', 'test')
+
+
+def test_parse_arguments_allows_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_dashed_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_restore():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'restore', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_dashed_restore():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--restore', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', 'list', '--archive', 'test')
+
+
+def test_parse_arguments_requires_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', 'extract')
+
+
+def test_parse_arguments_requires_archive_with_restore():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--config', 'myconfig', 'restore')
+
+
+def test_parse_arguments_allows_progress_before_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', 'create', 'list')
+
+
+def test_parse_arguments_allows_progress_after_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('create', '--progress', 'list')
+
+
+def test_parse_arguments_allows_progress_and_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list')
+
+
+def test_parse_arguments_allows_progress_and_restore():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', 'restore', '--archive', 'test', 'list')
+
+
+def test_parse_arguments_disallows_progress_without_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--progress', 'list')
+
+
+def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', 'create', 'list')
+
+
+def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', 'prune', 'list')
+
+
+def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--stats', 'list')
+
+
+def test_parse_arguments_allows_json_with_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('list', '--json')
+    module.parse_arguments('info', '--json')
+
+
+def test_parse_arguments_allows_json_with_dashed_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--info', '--json')
+
+
+def test_parse_arguments_disallows_json_with_both_list_and_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('list', 'info', '--json')
+
+
+def test_parse_arguments_check_only_extract_does_not_raise_extract_subparser_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('check', '--only', 'extract')
+
+
+def test_parse_arguments_extract_archive_check_does_not_raise_check_subparser_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('extract', '--archive', 'check')
+
+
+def test_parse_arguments_extract_with_check_only_extract_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract')

tests/integration/commands/test_borgmatic.py

@@ -1,103 +1,14 @@
+import subprocess
+
 from flexmock import flexmock
-import pytest
 
 from borgmatic.commands import borgmatic as module
 
 
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments()
-
-    assert parser.config_paths == config_paths
-    assert parser.excludes_filename is None
-    assert parser.verbosity is 0
-    assert parser.json is False
-
-
-def test_parse_arguments_with_path_arguments_overrides_defaults():
+def test_borgmatic_version_matches_news_version():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
-    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+    borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
+    news_version = open('NEWS').readline()
 
-    assert parser.config_paths == ['myconfig']
-    assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity is 0
-
-
-def test_parse_arguments_with_multiple_config_paths_parses_as_list():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
-
-    assert parser.config_paths == ['myconfig', 'otherconfig']
-    assert parser.verbosity is 0
-
-
-def test_parse_arguments_with_verbosity_flag_overrides_default():
-    config_paths = ['default']
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
-
-    parser = module.parse_arguments('--verbosity', '1')
-
-    assert parser.config_paths == config_paths
-    assert parser.excludes_filename is None
-    assert parser.verbosity == 1
-
-
-def test_parse_arguments_with_json_flag_overrides_default():
-    parser = module.parse_arguments('--list', '--json')
-    assert parser.json is True
-
-
-def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments()
-
-    assert parser.prune is True
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--prune')
-
-    assert parser.prune is True
-    assert parser.create is False
-    assert parser.check is False
-
-
-def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    parser = module.parse_arguments('--create', '--check')
-
-    assert parser.prune is False
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_invalid_arguments_exits():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(SystemExit):
-        module.parse_arguments('--posix-me-harder')
-
-
-def test_parse_arguments_with_json_flag_with_list_or_info_flag_does_not_raise_any_error():
-    module.parse_arguments('--list', '--json')
-    module.parse_arguments('--info', '--json')
-
-
-def test_parse_arguments_with_json_flag_but_no_list_or_info_flag_raises_value_error():
-    with pytest.raises(ValueError):
-        module.parse_arguments('--json')
-
-
-def test_parse_arguments_with_json_flag_and_both_list_and_info_flag_raises_value_error():
-    with pytest.raises(ValueError):
-        module.parse_arguments('--list', '--info', '--json')
+    assert borgmatic_version == news_version

tests/integration/commands/test_convert_config.py

@@ -1,7 +1,7 @@
 import os
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.commands import convert_config as module
 

tests/integration/commands/test_validate_config.py

@@ -0,0 +1,20 @@
+from flexmock import flexmock
+
+from borgmatic.commands import validate_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_paths == config_paths
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    assert parser.config_paths == ['myconfig', 'otherconfig']

tests/integration/config/test_generate.py

@@ -1,9 +1,9 @@
-from io import StringIO
 import os
 import sys
+from io import StringIO
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import generate as module
 
@@ -31,19 +31,29 @@ def test_comment_out_line_skips_already_commented_out_line():
 def test_comment_out_line_comments_section_name():
     line = 'figgy-pudding:'
 
-    assert module._comment_out_line(line) == '#' + line
+    assert module._comment_out_line(line) == '# ' + line
 
 
 def test_comment_out_line_comments_indented_option():
     line = '    enabled: true'
 
-    assert module._comment_out_line(line) == '    #enabled: true'
+    assert module._comment_out_line(line) == '    # enabled: true'
+
+
+def test_comment_out_line_comments_twice_indented_option():
+    line = '        - item'
+
+    assert module._comment_out_line(line) == '        # - item'
 
 
 def test_comment_out_optional_configuration_comments_optional_config_only():
-    flexmock(module)._comment_out_line = lambda line: '#' + line
+    # The "# COMMENT_OUT" comment is a sentinel used to express that the following key is optional.
+    # It's stripped out of the final output.
+    flexmock(module)._comment_out_line = lambda line: '# ' + line
     config = '''
+# COMMENT_OUT
 foo:
+    # COMMENT_OUT
     bar:
         - baz
         - quux
@@ -53,30 +63,34 @@ location:
         - one
         - two
 
+    # This comment should be kept.
+    # COMMENT_OUT
     other: thing
     '''
 
+    # flake8: noqa
     expected_config = '''
-#foo:
-#    bar:
-#        - baz
-#        - quux
-#
+# foo:
+#     bar:
+#         - baz
+#         - quux
+
 location:
     repositories:
         - one
         - two
-#
-#    other: thing
+
+    # This comment should be kept.
+#     other: thing
     '''
 
     assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
 
 
-def test_render_configuration_does_not_raise():
-    flexmock(module.yaml).should_receive('round_trip_dump')
+def test_render_configuration_converts_configuration_to_yaml_string():
+    yaml_string = module._render_configuration({'foo': 'bar'})
 
-    module._render_configuration({})
+    assert yaml_string == 'foo: bar\n'
 
 
 def test_write_configuration_does_not_raise():
@@ -106,20 +120,96 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
     module.write_configuration('config.yaml', 'config: yaml')
 
 
-def test_add_comments_to_configuration_does_not_raise():
+def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
+    config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
+    schema = {'seq': [{'type': 'str'}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
+    config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
+    schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
+    config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
+    schema = {'seq': [{'map': {'foo': {}}}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_map_does_not_raise():
     # Ensure that it can deal with fields both in the schema and missing from the schema.
     config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
     schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
 
-    module.add_comments_to_configuration(config, schema)
+    module.add_comments_to_configuration_map(config, schema)
+
+
+def test_add_comments_to_configuration_map_with_skip_first_does_not_raise():
+    config = module.yaml.comments.CommentedMap([('foo', 33)])
+    schema = {'map': {'foo': {'desc': 'Foo'}}}
+
+    module.add_comments_to_configuration_map(config, schema, skip_first=True)
+
+
+def test_remove_commented_out_sentinel_keeps_other_comments():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.\nCOMMENT_OUT')
+
+    module.remove_commented_out_sentinel(config, field_name)
+
+    comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX]
+    assert len(comments) == 1
+    assert comments[0].value == '# Actual comment.\n'
+
+
+def test_remove_commented_out_sentinel_without_sentinel_keeps_other_comments():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
+
+    module.remove_commented_out_sentinel(config, field_name)
+
+    comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX]
+    assert len(comments) == 1
+    assert comments[0].value == '# Actual comment.\n'
+
+
+def test_remove_commented_out_sentinel_on_unknown_field_does_not_raise():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
+
+    module.remove_commented_out_sentinel(config, 'unknown')
 
 
 def test_generate_sample_configuration_does_not_raise():
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module.yaml).should_receive('round_trip_load')
     flexmock(module).should_receive('_schema_to_sample_configuration')
+    flexmock(module).should_receive('merge_source_configuration_into_destination')
     flexmock(module).should_receive('_render_configuration')
     flexmock(module).should_receive('_comment_out_optional_configuration')
     flexmock(module).should_receive('write_configuration')
 
-    module.generate_sample_configuration('config.yaml', 'schema.yaml')
+    module.generate_sample_configuration(None, 'dest.yaml', 'schema.yaml')
+
+
+def test_generate_sample_configuration_with_source_filename_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module.yaml).should_receive('round_trip_load')
+    flexmock(module.load).should_receive('load_configuration')
+    flexmock(module).should_receive('_schema_to_sample_configuration')
+    flexmock(module).should_receive('merge_source_configuration_into_destination')
+    flexmock(module).should_receive('_render_configuration')
+    flexmock(module).should_receive('_comment_out_optional_configuration')
+    flexmock(module).should_receive('write_configuration')
+
+    module.generate_sample_configuration('source.yaml', 'dest.yaml', 'schema.yaml')

tests/integration/config/test_legacy.py

@@ -1,7 +1,6 @@
-from io import StringIO
-
-from collections import OrderedDict
 import string
+from collections import OrderedDict
+from io import StringIO
 
 from borgmatic.config import legacy as module
 

tests/integration/config/test_load.py

@@ -0,0 +1,62 @@
+import sys
+
+import pytest
+import ruamel.yaml
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_load_configuration_parses_contents():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_inlines_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return('value')
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        'key: !include include.yaml'
+    )
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_merges_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        foo: bar
+        baz: quux
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: override
+        <<: !include include.yaml
+        '''
+    )
+
+    assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
+
+
+def test_load_configuration_does_not_merge_include_list():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+          - one
+          - two
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: bar
+        repositories:
+          <<: !include include.yaml
+        '''
+    )
+
+    with pytest.raises(ruamel.yaml.error.YAMLError):
+        assert module.load_configuration('config.yaml')

tests/integration/config/test_validate.py

@@ -2,8 +2,8 @@ import io
 import string
 import sys
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import validate as module
 
@@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
     module.parse_configuration('config.yaml', 'schema.yaml')
 
 
+def test_parse_configuration_inlines_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24},
+    }
+
+
+def test_parse_configuration_merges_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_daily: 1
+            <<: !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 1, 'keep_hourly': 24},
+    }
+
+
 def test_parse_configuration_raises_for_missing_config_file():
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')

tests/integration/test_execute.py

@@ -0,0 +1,80 @@
+import logging
+import subprocess
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic import execute as module
+
+
+def test_log_output_logs_each_line_separately():
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+
+    hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
+    module.log_output(
+        ['echo', 'hi'],
+        hi_process,
+        hi_process.stdout,
+        output_log_level=logging.INFO,
+        error_on_warnings=False,
+    )
+
+    there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
+    module.log_output(
+        ['echo', 'there'],
+        there_process,
+        there_process.stdout,
+        output_log_level=logging.INFO,
+        error_on_warnings=False,
+    )
+
+
+def test_log_output_includes_error_output_in_exception():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+
+    process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.log_output(
+            ['grep'],
+            process,
+            process.stdout,
+            output_log_level=logging.INFO,
+            error_on_warnings=False,
+        )
+
+    assert error.value.returncode == 2
+    assert error.value.output
+
+
+def test_log_output_truncates_long_error_output():
+    flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+
+    process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.log_output(
+            ['grep'],
+            process,
+            process.stdout,
+            output_log_level=logging.INFO,
+            error_on_warnings=False,
+        )
+
+    assert error.value.returncode == 2
+    assert error.value.output.startswith('...')
+
+
+def test_log_output_with_no_output_logs_nothing():
+    flexmock(module.logger).should_receive('log').never()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+
+    process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    module.log_output(
+        ['true'], process, process.stdout, output_log_level=logging.INFO, error_on_warnings=False
+    )

tests/integration/test_version.py

@@ -1,8 +0,0 @@
-import subprocess
-
-
-def test_setup_version_matches_news_version():
-    setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii')
-    news_version = open('NEWS').readline()
-
-    assert setup_version == news_version

tests/unit/borg/test_check.py

@@ -1,22 +1,19 @@
-from subprocess import STDOUT
 import logging
-import sys
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.borg import check as module
+
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+def insert_execute_command_mock(command):
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
-def insert_subprocess_never():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
+def insert_execute_command_never():
+    flexmock(module).should_receive('execute_command').never()
 
 
 def test_parse_checks_returns_them_as_tuple():
@@ -37,32 +34,76 @@ def test_parse_checks_with_blank_value_returns_defaults():
     assert checks == module.DEFAULT_CHECKS
 
 
+def test_parse_checks_with_none_value_returns_defaults():
+    checks = module._parse_checks({'checks': None})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
 def test_parse_checks_with_disabled_returns_no_checks():
     checks = module._parse_checks({'checks': ['disabled']})
 
     assert checks == ()
 
 
+def test_parse_checks_with_data_check_also_injects_archives():
+    checks = module._parse_checks({'checks': ['data']})
+
+    assert checks == ('data', 'archives')
+
+
+def test_parse_checks_with_data_check_passes_through_archives():
+    checks = module._parse_checks({'checks': ['data', 'archives']})
+
+    assert checks == ('data', 'archives')
+
+
+def test_parse_checks_prefers_override_checks_to_configured_checks():
+    checks = module._parse_checks({'checks': ['archives']}, only_checks=['repository', 'extract'])
+
+    assert checks == ('repository', 'extract')
+
+
+def test_parse_checks_with_override_data_check_also_injects_archives():
+    checks = module._parse_checks({'checks': ['extract']}, only_checks=['data'])
+
+    assert checks == ('data', 'archives')
+
+
 def test_make_check_flags_with_repository_check_returns_flag():
     flags = module._make_check_flags(('repository',))
 
     assert flags == ('--repository-only',)
 
 
+def test_make_check_flags_with_archives_check_returns_flag():
+    flags = module._make_check_flags(('archives',))
+
+    assert flags == ('--archives-only',)
+
+
+def test_make_check_flags_with_data_check_returns_flag():
+    flags = module._make_check_flags(('data',))
+
+    assert flags == ('--verify-data',)
+
+
 def test_make_check_flags_with_extract_omits_extract_flag():
     flags = module._make_check_flags(('extract',))
 
     assert flags == ()
 
 
-def test_make_check_flags_with_default_checks_returns_default_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+def test_make_check_flags_with_default_checks_and_default_prefix_returns_default_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, prefix=module.DEFAULT_PREFIX)
 
     assert flags == ('--prefix', module.DEFAULT_PREFIX)
 
 
-def test_make_check_flags_with_all_checks_returns_default_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS + ('extract',))
+def test_make_check_flags_with_all_checks_and_default_prefix_returns_default_flags():
+    flags = module._make_check_flags(
+        module.DEFAULT_CHECKS + ('extract',), prefix=module.DEFAULT_PREFIX
+    )
 
     assert flags == ('--prefix', module.DEFAULT_PREFIX)
 
@@ -70,7 +111,7 @@ def test_make_check_flags_with_all_checks_returns_default_flags():
 def test_make_check_flags_with_archives_check_and_last_includes_last_flag():
     flags = module._make_check_flags(('archives',), check_last=3)
 
-    assert flags == ('--archives-only', '--last', '3', '--prefix', module.DEFAULT_PREFIX)
+    assert flags == ('--archives-only', '--last', '3')
 
 
 def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
@@ -82,7 +123,7 @@ def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
 def test_make_check_flags_with_default_checks_and_last_includes_last_flag():
     flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
 
-    assert flags == ('--last', '3', '--prefix', module.DEFAULT_PREFIX)
+    assert flags == ('--last', '3')
 
 
 def test_make_check_flags_with_archives_check_and_prefix_includes_prefix_flag():
@@ -91,6 +132,18 @@ def test_make_check_flags_with_archives_check_and_prefix_includes_prefix_flag():
     assert flags == ('--archives-only', '--prefix', 'foo-')
 
 
+def test_make_check_flags_with_archives_check_and_empty_prefix_omits_prefix_flag():
+    flags = module._make_check_flags(('archives',), prefix='')
+
+    assert flags == ('--archives-only',)
+
+
+def test_make_check_flags_with_archives_check_and_none_prefix_omits_prefix_flag():
+    flags = module._make_check_flags(('archives',), prefix=None)
+
+    assert flags == ('--archives-only',)
+
+
 def test_make_check_flags_with_repository_check_and_prefix_omits_prefix_flag():
     flags = module._make_check_flags(('repository',), prefix='foo-')
 
@@ -117,12 +170,9 @@ def test_check_archives_calls_borg_with_parameters(checks):
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(('borg', 'check', 'repo'), stdout=stdout, stderr=STDOUT)
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -136,7 +186,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').never()
     flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
-    insert_subprocess_never()
+    insert_execute_command_never()
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -149,7 +199,7 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_logging_mock(logging.INFO)
-    insert_subprocess_mock(('borg', 'check', 'repo', '--info'), stdout=None, stderr=STDOUT)
+    insert_execute_command_mock(('borg', 'check', '--info', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -162,9 +212,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_logging_mock(logging.DEBUG)
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--debug', '--show-rc'), stdout=None, stderr=STDOUT
-    )
+    insert_execute_command_mock(('borg', 'check', '--debug', '--show-rc', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -174,7 +222,7 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter():
 def test_check_archives_without_any_checks_bails():
     consistency_config = {'check_last': None}
     flexmock(module).should_receive('_parse_checks').and_return(())
-    insert_subprocess_never()
+    insert_execute_command_never()
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
@@ -187,12 +235,9 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(('borg1', 'check', 'repo'), stdout=stdout, stderr=STDOUT)
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg1', 'check', 'repo'))
 
     module.check_archives(
         repository='repo',
@@ -208,14 +253,9 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--remote-path', 'borg1'), stdout=stdout, stderr=STDOUT
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo'))
 
     module.check_archives(
         repository='repo',
@@ -231,14 +271,9 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--lock-wait', '5'), stdout=stdout, stderr=STDOUT
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={'lock_wait': 5}, consistency_config=consistency_config
@@ -254,11 +289,7 @@ def test_check_archives_with_retention_prefix():
     flexmock(module).should_receive('_make_check_flags').with_args(
         checks, check_last, prefix
     ).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(('borg', 'check', 'repo'), stdout=stdout, stderr=STDOUT)
-
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
+    insert_execute_command_mock(('borg', 'check', 'repo'))
 
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config

tests/unit/borg/test_create.py

@@ -1,58 +1,13 @@
 import logging
-import os
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import create as module
+
 from ..test_verbosity import insert_logging_mock
 
 
-def test_initialize_environment_with_passcommand_should_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize_environment({'encryption_passcommand': 'command'})
-        assert os.environ.get('BORG_PASSCOMMAND') == 'command'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_environment_with_passphrase_should_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize_environment({'encryption_passphrase': 'pass'})
-        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_environment_with_ssh_command_should_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize_environment({'ssh_command': 'ssh -C'})
-        assert os.environ.get('BORG_RSH') == 'ssh -C'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_environment_without_configuration_should_not_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize_environment({})
-        assert os.environ.get('BORG_PASSCOMMAND') is None
-        assert os.environ.get('BORG_PASSPHRASE') is None
-        assert os.environ.get('BORG_RSH') is None
-    finally:
-        os.environ = orig_environ
-
-
 def test_expand_directory_with_basic_path_passes_it_through():
     flexmock(module.os.path).should_receive('expanduser').and_return('foo')
     flexmock(module.glob).should_receive('glob').and_return([])
@@ -90,6 +45,21 @@ def test_expand_directories_considers_none_as_no_directories():
     assert paths == ()
 
 
+def test_expand_home_directories_expands_tildes():
+    flexmock(module.os.path).should_receive('expanduser').with_args('~/bar').and_return('/foo/bar')
+    flexmock(module.os.path).should_receive('expanduser').with_args('baz').and_return('baz')
+
+    paths = module._expand_home_directories(('~/bar', 'baz'))
+
+    assert paths == ('/foo/bar', 'baz')
+
+
+def test_expand_home_directories_considers_none_as_no_directories():
+    paths = module._expand_home_directories(None)
+
+    assert paths == ()
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
     flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@@ -101,11 +71,6 @@ def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
     module._write_pattern_file([])
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
 def test_make_pattern_flags_includes_pattern_filename_when_given():
     pattern_flags = module._make_pattern_flags(
         location_config={'patterns': ['R /', '- /var']}, pattern_filename='/tmp/patterns'
@@ -185,22 +150,64 @@ def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
     assert exclude_flags == ('--exclude-if-present', 'exclude_me')
 
 
+def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': True})
+
+    assert exclude_flags == ('--keep-exclude-tags',)
+
+
+def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': False})
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': True})
+
+    assert exclude_flags == ('--exclude-nodump',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': False})
+
+    assert exclude_flags == ()
+
+
 def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
     exclude_flags = module._make_exclude_flags(location_config={})
 
     assert exclude_flags == ()
 
 
+def test_borgmatic_source_directories_set_when_directory_exists():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.os.path).should_receive('expanduser')
+
+    assert module.borgmatic_source_directories() == [module.BORGMATIC_SOURCE_DIRECTORY]
+
+
+def test_borgmatic_source_directories_empty_when_directory_does_not_exist():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os.path).should_receive('expanduser')
+
+    assert module.borgmatic_source_directories() == []
+
+
 DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
+ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 
 def test_create_archive_calls_borg_with_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -216,13 +223,17 @@ def test_create_archive_calls_borg_with_parameters():
 
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
         flexmock(name='/tmp/patterns')
     ).and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + pattern_flags)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + pattern_flags + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -238,15 +249,17 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(
-        ('exclude',)
-    )
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
         flexmock(name='/tmp/excludes')
     )
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
-    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + exclude_flags + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -261,12 +274,17 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--list', '--filter', 'AME', '--info', '--stats'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--list', '--filter', 'AME-', '--info', '--stats') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
     insert_logging_mock(logging.INFO)
 
     module.create_archive(
@@ -281,13 +299,43 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     )
 
 
+def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
+    )
+    insert_logging_mock(logging.INFO)
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        json=True,
+    )
+
+
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        CREATE_COMMAND + ('--list', '--filter', 'AME', '--stats', '--debug', '--show-rc')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc')
+        + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -303,13 +351,42 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     )
 
 
+def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
+    )
+    insert_logging_mock(logging.DEBUG)
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        json=True,
+    )
+
+
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--dry-run',))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--dry-run') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=True,
@@ -324,14 +401,20 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
 
 
 def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter():
-    """ --dry-run and --stats are mutually exclusive, see:
-    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    # --dry-run and --stats are mutually exclusive, see:
+    # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--list', '--filter', 'AME', '--info', '--dry-run'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--list', '--filter', 'AME-', '--info', '--dry-run')
+        + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
     insert_logging_mock(logging.INFO)
 
     module.create_archive(
@@ -347,15 +430,19 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
 
 
 def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter():
-    """ --dry-run and --stats are mutually exclusive, see:
-    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description """
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    # --dry-run and --stats are mutually exclusive, see:
+    # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        CREATE_COMMAND + ('--list', '--filter', 'AME', '--debug', '--show-rc', '--dry-run')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run')
+        + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -372,11 +459,16 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
 
 
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--checkpoint-interval', '600'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--checkpoint-interval', '600') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
 
     module.create_archive(
         dry_run=False,
@@ -390,12 +482,41 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     )
 
 
-def test_create_archive_with_compression_calls_borg_with_compression_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--chunker-params', '1,2,3,4') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'chunker_params': '1,2,3,4'},
+    )
+
+
+def test_create_archive_with_compression_calls_borg_with_compression_parameters():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--compression', 'rle') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
 
     module.create_archive(
         dry_run=False,
@@ -410,11 +531,16 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--remote-ratelimit', '100') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
 
     module.create_archive(
         dry_run=False,
@@ -428,12 +554,16 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     )
 
 
-def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--one-file-system') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -448,12 +578,40 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     )
 
 
-def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--read-special',))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--numeric-owner') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'numeric_owner': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -468,12 +626,17 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     )
 
 
-def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -481,19 +644,25 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': True,
+            option_name: True,
             'exclude_patterns': None,
         },
         storage_config={},
     )
 
 
-def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--nobsdflags',))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--no' + option_name.replace('_', '')) + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
 
     module.create_archive(
         dry_run=False,
@@ -501,7 +670,7 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': False,
+            option_name: False,
             'exclude_patterns': None,
         },
         storage_config={},
@@ -509,11 +678,16 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete
 
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--files-cache', 'ctime,size') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
 
     module.create_archive(
         dry_run=False,
@@ -529,11 +703,15 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg1',) + CREATE_COMMAND[1:])
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1', 'create') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -549,11 +727,16 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--remote-path', 'borg1') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+    )
 
     module.create_archive(
         dry_run=False,
@@ -569,11 +752,15 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--umask', '740') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -588,11 +775,15 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--lock-wait', '5') + ARCHIVE_WITH_PATHS, output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -606,14 +797,66 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     )
 
 
-def test_create_archive_with_json_calls_borg_with_json_parameter():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+def test_create_archive_with_stats_calls_borg_with_stats_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--json',))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--stats') + ARCHIVE_WITH_PATHS, output_log_level=logging.WARNING
+    )
 
     module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        stats=True,
+    )
+
+
+def test_create_archive_with_progress_calls_borg_with_progress_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command_without_capture').with_args(
+        ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        progress=True,
+    )
+
+
+def test_create_archive_with_json_calls_borg_with_json_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
+    ).and_return('[]')
+
+    json_output = module.create_archive(
         dry_run=False,
         repository='repo',
         location_config={
@@ -625,16 +868,46 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
         json=True,
     )
 
+    assert json_output == '[]'
 
-def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(
-        ()
-    )
+
+def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS, output_log_level=None
+    ).and_return('[]')
+
+    json_output = module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        json=True,
+        stats=True,
+    )
+
+    assert json_output == '[]'
+
+
+def test_create_archive_with_source_directories_glob_expands():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
+        output_log_level=logging.INFO,
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -651,11 +924,16 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'),
+        output_log_level=logging.INFO,
+    )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
@@ -671,14 +949,15 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(
-        ()
-    )
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
+        output_log_level=logging.INFO,
     )
 
     module.create_archive(
@@ -694,11 +973,15 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'), output_log_level=logging.INFO
+    )
 
     module.create_archive(
         dry_run=False,
@@ -713,11 +996,16 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([])
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'),
+        output_log_level=logging.INFO,
+    )
 
     module.create_archive(
         dry_run=False,

tests/unit/borg/test_environment.py

@@ -0,0 +1,62 @@
+import os
+
+from borgmatic.borg import environment as module
+
+
+def test_initialize_with_passcommand_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'encryption_passcommand': 'command'})
+        assert os.environ.get('BORG_PASSCOMMAND') == 'command'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_with_passphrase_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'encryption_passphrase': 'pass'})
+        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_with_ssh_command_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'ssh_command': 'ssh -C'})
+        assert os.environ.get('BORG_RSH') == 'ssh -C'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_without_configuration_should_only_set_default_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({})
+
+        assert {key: value for key, value in os.environ.items() if key.startswith('BORG_')} == {
+            'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no',
+            'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no',
+        }
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_with_relocated_repo_access_should_override_default():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'relocated_repo_access_is_ok': True})
+        assert os.environ.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
+    finally:
+        os.environ = orig_environ