Disable console color via "color" option in borgmatic configuration output section (#191).

Closes https://projects.torsion.org/witten/borgmatic/issues/178.

.drone.yml

@@ -0,0 +1,57 @@
+---
+kind: pipeline
+name: python-3-5-alpine-3-9
+
+steps:
+- name: build
+  image: python:3.5-alpine3.9
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-6-alpine-3-9
+
+steps:
+- name: build
+  image: python:3.6-alpine3.9
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-9
+
+steps:
+- name: build
+  image: python:3.7-alpine3.9
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: python-3-7-alpine-3-7
+
+steps:
+- name: build
+  image: python:3.7-alpine3.7
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
+name: documentation
+
+steps:
+- name: build
+  image: plugins/docker
+  settings:
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    repo: witten/borgmatic-docs
+    dockerfile: docs/Dockerfile
+  when:
+    branch:
+      - master

.eleventy.js

@@ -0,0 +1,41 @@
+const pluginSyntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
+const inclusiveLangPlugin = require("@11ty/eleventy-plugin-inclusive-language");
+
+module.exports = function(eleventyConfig) {
+    eleventyConfig.addPlugin(pluginSyntaxHighlight);
+    eleventyConfig.addPlugin(inclusiveLangPlugin);
+
+    let markdownIt = require("markdown-it");
+    let markdownItAnchor = require("markdown-it-anchor");
+    let markdownItReplaceLink = require("markdown-it-replace-link");
+
+    let markdownItOptions = {
+        html: true,
+        breaks: false,
+        linkify: true,
+        // Replace links to .md files with links to directories. This allows unparsed Markdown links
+        // to work on GitHub, while rendered links elsewhere also work.
+        replaceLink: function (link, env) {
+            return link.replace(/\.md$/, '/');
+        }
+    };
+    let markdownItAnchorOptions = {
+        permalink: true,
+        permalinkClass: "direct-link"
+    };
+
+    eleventyConfig.setLibrary(
+        "md",
+        markdownIt(markdownItOptions)
+            .use(markdownItAnchor, markdownItAnchorOptions)
+            .use(markdownItReplaceLink)
+    );
+
+    return {
+        templateFormats: [
+          "md",
+          "txt"
+        ]
+    }
+};
+

.gitea/issue_template.md

@@ -0,0 +1,31 @@
+#### What I'm trying to do and why
+
+#### Steps to reproduce (if a bug)
+
+Include (sanitized) borgmatic configuration files if applicable.
+
+#### Actual behavior (if a bug)
+
+Include (sanitized) `--verbosity 2` output if applicable.
+
+#### Expected behavior (if a bug)
+
+#### Other notes / implementation ideas
+
+#### Environment
+
+**borgmatic version:** [version here]
+
+Use `sudo borgmatic --version` or `sudo pip show borgmatic | grep ^Version`
+
+**borgmatic installation method:** [e.g., Debian package, Docker container, etc.]
+
+**Borg version:** [version here]
+
+Use `sudo borg --version`
+
+**Python version:** [version here]
+
+Use `python --version`
+
+**operating system and version:** [OS here]

.gitignore

@@ -1,6 +1,10 @@
 *.egg-info
 *.pyc
 *.swp
+.cache
+.coverage
+.pytest_cache
 .tox
-build
-dist
+build/
+dist/
+pip-wheel-metadata/

.hgignore

@@ -1,9 +0,0 @@
-syntax: glob
-*.egg-info
-*.pyc
-*.swp
-.cache
-.coverage
-.tox
-build
-dist

.hgtags

@@ -1,39 +0,0 @@
-467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
-7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
-4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
-7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
-cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
-38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
-83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
-acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
-0afff209b902698c2266986129d6dc9f5f913101 0.1.5
-4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
-5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
-977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
-0000000000000000000000000000000000000000 github/yaml_config_files
-28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master
-dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8
-0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0
-de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1
-9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2
-32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3
-5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0
-7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1
-f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2
-3f838f661546e04529b453aa443529b432afc243 1.1.3
-3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4
-64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5
-4daa944c122c572b9b56bfcac3f4e2869181c630 1.1.6

AUTHORS

@@ -1,7 +1,12 @@
 Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
+Florian Lindner: Logging rewrite
 Henning Schroeder: Copy editing
+Johannes Feichtner: Support for user hooks
 Michele Lazzeri: Custom archive names
-Robin `ypid` Schneider: Support additional options of Borg
+Nick Whyte: Support prefix filtering for archive consistency checks
+newtonne: Read encryption password from external file
+Robin `ypid` Schneider: Support additional options of Borg and add validate-borgmatic-config command
 Scott Squires: Custom archive names
+Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option

NEWS

@@ -1,37 +1,265 @@
+1.3.8
+ * #191: Disable console color via "color" option in borgmatic configuration output section.
+
+1.3.7
+ * #196: Fix for unclear error message for invalid YAML merge include.
+ * #197: Don't color syslog output.
+ * Change default syslog verbosity to show errors only.
+
+1.3.6
+ * #53: Log to syslog in addition to existing console logging. Add --syslog-verbosity flag to
+   customize the log level. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/
+ * #178: Look for .yml configuration file extension in addition to .yaml.
+ * #189: Set umask used when executing hooks via "umask" option in borgmatic hooks section.
+ * Remove Python cache files before each Tox run.
+ * Add #borgmatic Freenode IRC channel to documentation.
+ * Add Borg/borgmatic hosting providers section to documentation.
+ * Add files for building documentation into a Docker image for web serving.
+ * Upgrade project build server from Drone 0.8 to 1.1.
+ * Build borgmatic documentation during continuous integration.
+ * We're nearly at 500 ★s on GitHub. We can do this!
+
+1.3.5
+ * #153: Support for various Borg directory environment variables (BORG_CONFIG_DIR, BORG_CACHE_DIR,
+   etc.) via options in borgmatic's storage configuration.
+ * #177: Fix for regression with missing verbose log entries.
+
+1.3.4
+ * Part of #125: Color borgmatic (but not Borg) output when using an interactive terminal.
+ * #166: Run tests for all installed versions of Python.
+ * #168: Update README with continuous integration badge.
+ * #169: Automatically sort Python imports in code.
+ * Document installing borgmatic with pip install --user instead of a system Python install.
+ * Get more reproducible builds by pinning the versions of pip and tox used to run tests.
+ * Factor out build/test configuration from tox.ini file.
+
+1.3.3
+ * Add validate-borgmatic-config command, useful for validating borgmatic config generated by
+   configuration management or even edited by hand.
+
+1.3.2
+ * #160: Fix for hooks executing when using --dry-run. Now hooks are skipped during a dry run.
+
+1.3.1
+ * #155: Fix for invalid JSON output when using multiple borgmatic configuration files.
+ * #157: Fix for seemingly random filename ordering when running through a directory of
+   configuration files.
+ * Fix for empty JSON output when using --create --json.
+ * Now capturing Borg output only when --json flag is used. Previously, borgmatic delayed Borg
+   output even without the --json flag.
+
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
+1.2.18
+ * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
+   borgmatic's location section.
+
+1.2.17
+ * #140: List the files within an archive via --list --archive option.
+
+1.2.16
+ * #119: Include a sample borgmatic configuration file in the documentation.
+ * #123: Support for Borg archive restoration via borgmatic --extract command-line flag.
+ * Refactor documentation into multiple separate pages for clarity and findability.
+ * Organize options within command-line help into logical groups.
+ * Exclude tests from distribution packages.
+
+1.2.15
+ * #127: Remove date echo from schema example, as it's not a substitute for real logging.
+ * #132: Leave exclude_patterns glob expansion to Borg, since doing it in borgmatic leads to
+   confusing behavior.
+ * #136: Handle and format validation errors raised during argument parsing.
+ * #138: Allow use of --stats flag when --create or --prune flags are implied.
+
+1.2.14
+ * #103: When generating sample configuration with generate-borgmatic-config, document the defaults
+   for each option.
+ * #116: When running multiple configuration files, attempt all configuration files even if one of
+   them errors. Log a summary of results at the end.
+ * Add borgmatic --version command-line flag to get the current installed version number.
+
+1.2.13
+ * #100: Support for --stats command-line flag independent of --verbosity.
+ * #117: With borgmatic --init command-line flag, proceed without erroring if a repository already
+   exists.
+
+1.2.12
+ * #110: Support for Borg repository initialization via borgmatic --init command-line flag.
+ * #111: Update Borg create --filter values so a dry run lists files to back up.
+ * #113: Update README with link to a new/forked Docker image.
+ * Prevent deprecated --excludes command-line option from being used.
+ * Refactor README a bit to flow better for first-time users.
+ * Update README with a few additional borgmatic packages (Debian and Ubuntu).
+
+1.2.11
+ * #108: Support for Borg create --progress via borgmatic command-line flag.
+
+1.2.10
+ * #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
+   storage section.
+
+1.2.9
+ * #102: Fix for syntax error that occurred in Python 3.5 and below.
+ * Make automated tests support running in Python 3.5.
+
+1.2.8
+ * #73: Enable consistency checks for only certain repositories via "check_repositories" option in
+   borgmatic's consistency configuration. Handy for large repositories that take forever to check.
+ * Include link to issue tracker within various command output.
+ * Run continuous integration tests on a matrix of Python and Borg versions.
+
+1.2.7
+ * #98: Support for Borg --keep-secondly prune option.
+ * Use Black code formatter and Flake8 code checker as part of running automated tests.
+ * Add an end-to-end automated test that actually integrates with Borg.
+ * Set up continuous integration for borgmatic automated tests on projects.evoworx.org.
+
+1.2.6
+ * Fix generated configuration to also include a "keep_daily" value so pruning works out of the
+   box.
+
+1.2.5
+ * #57: When generating sample configuration with generate-borgmatic-config, comment out all
+   optional configuration so as to streamline the initial configuration process.
+
+1.2.4
+ * Fix for archive checking traceback due to parameter mismatch.
+
+1.2.3
+ * #64, #90, #92: Rewrite of logging system. Now verbosity flags passed to Borg are derived from
+   borgmatic's log level. Note that the output of borgmatic might slightly change.
+ * Part of #80: Support for Borg create --read-special via "read_special" option in borgmatic's
+   location configuration.
+ * #87: Support for Borg create --checkpoint-interval via "checkpoint_interval" option in
+   borgmatic's storage configuration.
+ * #88: Fix declared pykwalify compatibility version range in setup.py to prevent use of ancient
+   versions of pykwalify with large version numbers.
+ * #89: Pass --show-rc option to Borg when at highest verbosity level.
+ * #94: Support for Borg --json option via borgmatic command-line to --create archives.
+
+1.2.2
+ * #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in
+   borgmatic as a pykwalify RuleError.
+
+1.2.1
+ * Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info.
+ * #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/
+   path.
+ * #74, #83: Support for Borg --json option via borgmatic command-line to --list archives or show
+   archive --info in JSON format, ideal for programmatic consumption.
+ * #38, #76: Upgrade ruamel.yaml compatibility version range and fix support for Python 3.7.
+ * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
+   editor swap files, etc.
+ * #81: Document user-defined hooks run before/after backup, or on error.
+ * Add code style guidelines to the documention.
+
+1.2.0
+ * #61: Support for Borg --list option via borgmatic command-line to list all archives.
+ * #61: Support for Borg --info option via borgmatic command-line to display summary information.
+ * #62: Update README to mention other ways of installing borgmatic.
+ * Support for Borg --prefix option for consistency checks via "prefix" option in borgmatic's
+   consistency configuration.
+ * Add introductory screencast link to documentation.
+ * #59: Ignore "check_last" and consistency "prefix" when "archives" not in consistency checks.
+ * #60: Add "Persistent" flag to systemd timer example.
+ * #63: Support for Borg --nobsdflags option to skip recording bsdflags (e.g. NODUMP, IMMUTABLE) in
+   archive.
+ * #69: Support for Borg prune --umask option using value of existing "umask" option in borgmatic's
+   storage configuration.
+ * Update tox.ini to only assume Python 3.x instead of Python 3.4 specifically.
+ * Add ~/.config/borgmatic/config.yaml to default configuration path probing.
+ * Document how to develop on and contribute to borgmatic.
+
+1.1.15
+ * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
+ * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
+   Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
+ * #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
+ * #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
+ * #58: Support for using tilde in exclude_patterns to reference home directory.
+
+1.1.14
+ * #49: Fix for typo in --patterns-from option.
+ * #47: Support for Borg --dry-run option via borgmatic command-line.
+
+1.1.13
+ * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
+   "archives", and "extract") are specified in borgmatic configuration.
+ * #48: Add "local_path" to configuration for specifying an alternative Borg executable path.
+ * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
+   includes/excludes.
+ * Moved issue tracker from Taiga to integrated Gitea tracker at
+   https://projects.torsion.org/witten/borgmatic/issues
+
+1.1.12
+ * #46: Declare dependency on pykwalify 1.6 or above, as older versions yield "Unknown key: version"
+   rule errors.
+ * Support for Borg --keep-minutely prune option.
+
+1.1.11
+ * #26: Add "ssh_command" to configuration for specifying a custom SSH command or options.
+ * Fix for incorrect /etc/borgmatic.d/ configuration path probing on macOS. This problem manifested
+   as an error on startup: "[Errno 2] No such file or directory: '/etc/borgmatic.d'".
+
+1.1.10
+ * Pass several Unix signals through to child processes like Borg. This means that Borg now properly
+   shuts down if borgmatic is terminated (e.g. due to a system suspend).
+ * #30: Support for using tilde in repository paths to reference home directory.
+ * #43: Support for Borg --files-cache option for setting the files cache operation mode.
+ * #45: Support for Borg --remote-ratelimit option for limiting upload rate.
+ * Log invoked Borg commands when at highest verbosity level.
+
+1.1.9
+ * #17, #39: Support for user-defined hooks before/after backup, or on error.
+ * #34: Improve clarity of logging spew at high verbosity levels.
+ * #30: Support for using tilde in source directory path to reference home directory.
+ * Require "prefix" in retention section when "archive_name_format" is set. This is to avoid
+   accidental pruning of archives with a different archive name format. For similar reasons, default
+   "prefix" to "{hostname}-" if not specified.
+ * Convert main source repository from Mercurial to Git.
+ * Update dead links to Borg documentation.
+
+1.1.8
+ * #40: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default
+   config paths.
+
 1.1.7
 
- * #28: Add "archive_name_format" to configuration for customizing archive names.
+ * #29: Add "archive_name_format" to configuration for customizing archive names.
  * Fix for traceback when "exclude_from" value is empty in configuration file.
  * When pruning, make highest verbosity level list archives kept and pruned.
  * Clarification of Python 3 pip usage in documentation.
 
 1.1.6
 
- * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
+ * #13, #36: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
 
 1.1.5
 
- * #34: New "extract" consistency check that performs a dry-run extraction of the most recent
+ * #35: New "extract" consistency check that performs a dry-run extraction of the most recent
    archive.
 
 1.1.4
 
- * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or
+ * #18: Added command-line flags for performing a borgmatic run with only pruning, creating, or
    checking enabled. This supports use cases like running consistency checks from a different cron
    job with a different frequency, or running pruning with a different verbosity level.
 
 1.1.3
 
- * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
+ * #15: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
  * Fix for generate-borgmatic-config writing config with invalid one_file_system value.
 
 1.1.2
 
- * #32: Fix for passing check_last as integer to subprocess when calling Borg.
+ * #33: Fix for passing check_last as integer to subprocess when calling Borg.
 
 1.1.1
 
- * Part of #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
+ * Part of #33: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
    an integer.
  * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present.
 
@@ -40,8 +268,8 @@
  * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
  * Added generate-borgmatic-config command for initial config creation.
  * Dropped Python 2 support. Now Python 3 only.
- * #18: Fix for README mention of sample files not included in package.
- * #22: Sample files for triggering borgmatic from a systemd timer.
+ * #19: Fix for README mention of sample files not included in package.
+ * #23: Sample files for triggering borgmatic from a systemd timer.
  * Support for backing up to multiple repositories.
  * To free up space, now pruning backups prior to creating a new backup.
  * Enabled test coverage output during tox runs.
@@ -49,15 +277,15 @@
 
 1.0.3
 
- * #21: Fix for verbosity flag not actually causing verbose output.
+ * #22: Fix for verbosity flag not actually causing verbose output.
 
 1.0.2
 
- * #20: Fix for traceback when remote_path option is missing.
+ * #21: Fix for traceback when remote_path option is missing.
 
 1.0.1
 
- * #19: Support for Borg's --remote-path option to use an alternate Borg
+ * #20: Support for Borg's --remote-path option to use an alternate Borg
    executable. See sample/config.
 
 1.0.0
@@ -79,13 +307,13 @@
 
 0.1.7
 
- * #11: Fixed parsing of punctuation in configuration file.
+ * #12: Fixed parsing of punctuation in configuration file.
  * Better error message when configuration file is missing.
 
 0.1.6
 
- * #9: New configuration option for the encryption passphrase.
- * #10: Support for Borg's new archive compression feature.
+ * #10: New configuration option for the encryption passphrase.
+ * #11: Support for Borg's new archive compression feature.
 
 0.1.5
 
@@ -97,7 +325,7 @@
 
 0.1.3
 
- * #1: Add support for "borg check --last N" to Borg backend.
+ * #2: Add support for "borg check --last N" to Borg backend.
 
 0.1.2
 

README.md

@@ -1,13 +1,20 @@
-<img src="https://cdn.rawgit.com/witten/borgmatic/master/static/borgmatic.svg" width="150px" style="float: right; padding-left: 1em;">
+---
+title: borgmatic
+permalink: index.html
+---
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg)</a>
 
 ## Overview
 
-borgmatic (formerly atticmatic) is a simple Python wrapper script for the
-[Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that
-initiates a backup, prunes any old backups according to a retention policy,
-and validates backups for consistency. The script supports specifying your
-settings in a declarative configuration file rather than having to put them
-all on the command-line, and handles common errors.
+<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
+
+borgmatic is simple, configuration-driven backup software for servers and
+workstations. Backup all of your machines from the command-line or scheduled
+jobs. No GUI required. Built atop [Borg Backup](https://www.borgbackup.org/),
+borgmatic initiates a backup, prunes any old backups according to a retention
+policy, and validates backups for consistency. borgmatic supports specifying
+your settings in a declarative configuration file, rather than having to put
+them all on the command-line, and handles common errors.
 
 Here's an example config file:
 
@@ -41,258 +48,76 @@ consistency:
 ```
 
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://torsion.org/hg/borgmatic). It's also mirrored on
-[GitHub](https://github.com/witten/borgmatic) and
-[BitBucket](https://bitbucket.org/dhelfman/borgmatic) for convenience.
+available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
+on [GitHub](https://github.com/witten/borgmatic) for convenience.
 
+Want to see borgmatic in action? Check out the <a
+href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 
-## Installation
+<script src="https://asciinema.org/a/203761.js" id="asciicast-203761" async></script>
 
-To get up and running, follow the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host. Note that if you plan to run
-borgmatic on a schedule with cron, and you encrypt your Borg repository with
-a passphrase instead of a key file, you'll need to set the borgmatic
-`encryption_passphrase` configuration variable. See the repository encryption
-section of the Quick Start for more info.
 
-If the repository is on a remote host, make sure that your local root user has
-key-based ssh access to the desired user account on the remote host.
+## How-to guides
 
-To install borgmatic, run the following command to download and install it:
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) ⬅ *Start here!*
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
+ * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
+ * [Run preparation steps before backups](https://torsion.org/borgmatic/docs/how-to/run-preparation-steps-before-backups/)
+ * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 
-    sudo pip3 install --upgrade borgmatic
 
-Note that your pip binary may have a different name than "pip3". Make sure
-you're using Python 3, as borgmatic does not support Python 2.
+## Reference guides
 
-## Configuration
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)
 
-After you install borgmatic, generate a sample configuration file:
 
-    sudo generate-borgmatic-config
+## Hosting providers
 
-This generates a sample configuration file at /etc/borgmatic/config.yaml (by
-default). You should edit the file to suit your needs, as the values are just
-representative. All fields are optional except where indicated, so feel free
-to remove anything you don't need.
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
 
-You can also have a look at the [full configuration
-schema](https://torsion.org/hg/borgmatic/file/tip/borgmatic/config/schema.yaml)
-for the authoritative set of all configuration options. This is handy if
-borgmatic has added new options since you originally created your
-configuration file.
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
 
+## Support and contributing
 
-### Multiple configuration files
+### Issues
 
-A more advanced usage is to create multiple separate configuration files and
-place each one in an /etc/borgmatic.d directory. For instance:
+You've got issues? Or an idea for a feature enhancement? We've got an [issue
+tracker](https://projects.torsion.org/witten/borgmatic/issues). In order to
+create a new issue or comment on an issue, you'll need to [login
+first](https://projects.torsion.org/user/login). Note that you can login with
+an existing GitHub account if you prefer.
 
-    sudo mkdir /etc/borgmatic.d
-    sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
-    sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
-
-With this approach, you can have entirely different backup policies for
-different applications on your system. For instance, you may want one backup
-configuration for your database data directory, and a different configuration
-for your user home directories.
-
-When you set up multiple configuration files like this, borgmatic will run
-each one in turn from a single borgmatic invocation. This includes, by
-default, the traditional /etc/borgmatic/config.yaml as well.
-
-
-## Upgrading
-
-In general, all you should need to do to upgrade borgmatic is run the
-following:
-
-    sudo pip3 install --upgrade borgmatic
-
-However, see below about special cases.
-
-
-### Upgrading from borgmatic 1.0.x
-
-borgmatic changed its configuration file format in version 1.1.0 from
-INI-style to YAML. This better supports validation, and has a more natural way
-to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic.
-
-As of version 1.1.0, borgmatic no longer supports Python 2. If you were
-already running borgmatic with Python 3, then you can simply upgrade borgmatic
-in-place:
-
-    sudo pip3 install --upgrade borgmatic
-
-But if you were running borgmatic with Python 2, uninstall and reinstall instead:
-
-    sudo pip uninstall borgmatic
-    sudo pip3 install borgmatic
-
-The pip binary names for different versions of Python can differ, so the above
-commands may need some tweaking to work on your machine.
-
-
-Once borgmatic is upgraded, run:
-
-    sudo upgrade-borgmatic-config
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and
-excludes files. The new version of borgmatic will consume the YAML
-configuration file instead of the old one.
-
-
-### Upgrading from atticmatic
-
-You can ignore this section if you're not an atticmatic user (the former name
-of borgmatic).
-
-borgmatic only supports Borg now and no longer supports Attic. So if you're
-an Attic user, consider switching to Borg. See the [Borg upgrade
-command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
-for more information. Then, follow the instructions above about setting up
-your borgmatic configuration files.
-
-If you were already using Borg with atticmatic, then you can easily upgrade
-from atticmatic to borgmatic. Simply run the following commands:
-
-    sudo pip3 uninstall atticmatic
-    sudo pip3 install borgmatic
-
-That's it! borgmatic will continue using your /etc/borgmatic configuration
-files.
-
-
-## Usage
-
-You can run borgmatic and start a backup simply by invoking it without
-arguments:
-
-    borgmatic
-
-This will also prune any old backups as per the configured retention policy,
-and check backups for consistency problems due to things like file damage.
-
-If you'd like to see the available command-line arguments, view the help:
-
-    borgmatic --help
-
-Note that borgmatic prunes archives *before* creating an archive, so as to
-free up space for archiving. This means that when a borgmatic run finishes,
-there may still be prune-able archives. Not to worry, as they will get cleaned
-up at the start of the next run.
-
-### Verbosity
-
-By default, the backup will proceed silently except in the case of errors. But
-if you'd like to to get additional information about the progress of the
-backup as it proceeds, use the verbosity option:
-
-    borgmatic --verbosity 1
-
-Or, for even more progress spew:
-
-    borgmatic --verbosity 2
-
-### À la carte
-
-If you want to run borgmatic with only pruning, creating, or checking enabled,
-the following optional flags are available:
-
-    borgmatic --prune
-    borgmatic --create
-    borgmatic --check
-
-You can run with only one of these flags provided, or you can mix and match
-any number of them. This supports use cases like running consistency checks
-from a different cron job with a different frequency, or running pruning with
-a different verbosity level.
-
-
-## Autopilot
-
-If you want to run borgmatic automatically, say once a day, the you can
-configure a job runner to invoke it periodically.
-
-### cron
-
-If you're using cron, download the [sample cron
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/cron/borgmatic).
-Then, from the directory where you downloaded it:
-
-    sudo mv borgmatic /etc/cron.d/borgmatic
-    sudo chmod +x /etc/cron.d/borgmatic
-
-You can modify the cron file if you'd like to run borgmatic more or less frequently.
-
-### systemd
-
-If you're using systemd instead of cron to run jobs, download the [sample
-systemd service
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.service)
-and the [sample systemd timer
-file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.timer).
-Then, from the directory where you downloaded them:
-
-    sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
-    sudo systemctl enable borgmatic.timer
-    sudo systemctl start borgmatic.timer
-
-Feel free to modify the timer file based on how frequently you'd like
-borgmatic to run.
-
-
-## Running tests
-
-First install tox, which is used for setting up testing environments:
-
-    pip3 install tox
-
-Then, to actually run tests, run:
-
-    tox
-
-
-## Troubleshooting
-
-### Broken pipe with remote repository
-
-When running borgmatic on a large remote repository, you may receive errors
-like the following, particularly while "borg check" is validating backups for
-consistency:
-
-    Write failed: Broken pipe
-    borg: Error: Connection closed by remote host
-
-This error can be caused by an ssh timeout, which you can rectify by adding
-the following to the ~/.ssh/config file on the client:
-
-    Host *
-        ServerAliveInterval 120
-
-This should make the client keep the connection alive while validating
-backups.
-
-
-### libyaml compilation errors
-
-borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
-use a C YAML library (libyaml) if present. But if it's not installed, then
-when installing or upgrading borgmatic, you may see errors about compiling the
-YAML library. If so, not to worry. borgmatic should install and function
-correctly even without the C YAML library. And borgmatic won't be any faster
-with the C library present, so you don't need to go out of your way to install
-it.
-
-
-## Issues and feedback
-
-Got an issue or an idea for a feature enhancement? Check out the [borgmatic
-issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues?page=1&status=399951,399952,399955). In
-order to create a new issue or comment on an issue, you'll need to [login
-first](https://tree.taiga.io/login).
+If you'd like to chat with borgmatic developers or users, head on over to the
+`#borgmatic` IRC channel on Freenode, either via <a
+href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
+native <a href="irc://chat.freenode.net:6697">IRC client</a>.
 
 Other questions or comments? Contact <mailto:witten@torsion.org>.
+
+
+### Contributing
+
+If you'd like to contribute to borgmatic development, please feel free to
+submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)
+or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
+to discuss your idea. We also accept Pull Requests on GitHub, if that's more
+your thing. In general, contributions are very welcome. We don't bite! 
+
+Also, please check out the [borgmatic development
+how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
+info on cloning source code, running tests, etc.
+
+<script>
+  var links = document.getElementsByClassName("referral");
+  links[Math.floor(Math.random() * links.length)].style.display = "none";
+</script>

borgmatic/borg/check.py

@@ -1,11 +1,13 @@
-import os
-import subprocess
+import logging
 
 from borgmatic.borg import extract
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
+from borgmatic.execute import execute_command
 
 DEFAULT_CHECKS = ('repository', 'archives')
+DEFAULT_PREFIX = '{hostname}-'
+
+
+logger = logging.getLogger(__name__)
 
 
 def _parse_checks(consistency_config):
@@ -27,10 +29,12 @@ def _parse_checks(consistency_config):
     if checks == ['disabled']:
         return ()
 
-    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
+    return (
+        tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
+    )
 
 
-def _make_check_flags(checks, check_last=None):
+def _make_check_flags(checks, check_last=None, prefix=None):
     '''
     Given a parsed sequence of checks, transform it into tuple of command-line flags.
 
@@ -42,44 +46,73 @@ def _make_check_flags(checks, check_last=None):
 
         ('--repository-only',)
 
-    Additionally, if a check_last value is given, a "--last" flag will be added.
+    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    flags because Borg does both checks by default.
+
+    Additionally, if a check_last value is given and "archives" is in checks, then include a
+    "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
+    "--prefix" flag.
     '''
-    last_flag = ('--last', str(check_last)) if check_last else ()
-    if checks == DEFAULT_CHECKS:
-        return last_flag
+    if 'archives' in checks:
+        last_flags = ('--last', str(check_last)) if check_last else ()
+        prefix_flags = ('--prefix', prefix) if prefix else ('--prefix', DEFAULT_PREFIX)
+    else:
+        last_flags = ()
+        prefix_flags = ()
+        if check_last:
+            logger.warning(
+                'Ignoring check_last option, as "archives" is not in consistency checks.'
+            )
+        if prefix:
+            logger.warning(
+                'Ignoring consistency prefix option, as "archives" is not in consistency checks.'
+            )
 
-    return tuple(
-        '--{}-only'.format(check) for check in checks
-        if check in DEFAULT_CHECKS
-    ) + last_flag
+    if set(DEFAULT_CHECKS).issubset(set(checks)):
+        return last_flags + prefix_flags
+
+    return (
+        tuple('--{}-only'.format(check) for check in checks if check in DEFAULT_CHECKS)
+        + last_flags
+        + prefix_flags
+    )
 
 
-def check_archives(verbosity, repository, consistency_config, remote_path=None):
+def check_archives(
+    repository, storage_config, consistency_config, local_path='borg', remote_path=None
+):
     '''
-    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    command to run, check the contained Borg archives for consistency.
+    Given a local or remote repository path, a storage config dict, a consistency config dict,
+    and a local/remote commands to run, check the contained Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config)
     check_last = consistency_config.get('check_last', None)
+    lock_wait = None
 
     if set(checks).intersection(set(DEFAULT_CHECKS)):
         remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-        verbosity_flags = {
-            VERBOSITY_SOME: ('--info',),
-            VERBOSITY_LOTS: ('--debug',),
-        }.get(verbosity, ())
+        lock_wait = storage_config.get('lock_wait', None)
+        lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
+
+        verbosity_flags = ()
+        if logger.isEnabledFor(logging.INFO):
+            verbosity_flags = ('--info',)
+        if logger.isEnabledFor(logging.DEBUG):
+            verbosity_flags = ('--debug', '--show-rc')
+
+        prefix = consistency_config.get('prefix')
 
         full_command = (
-            'borg', 'check',
-            repository,
-        ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
+            (local_path, 'check', repository)
+            + _make_check_flags(checks, check_last, prefix)
+            + remote_path_flags
+            + lock_wait_flags
+            + verbosity_flags
+        )
 
-        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-        stdout = None if verbosity_flags else open(os.devnull, 'w')
-
-        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+        execute_command(full_command)
 
     if 'extract' in checks:
-        extract.extract_last_archive_dry_run(verbosity, repository, remote_path)
+        extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -1,46 +1,90 @@
 import glob
 import itertools
+import logging
 import os
-import subprocess
 import tempfile
 
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
 
 
-def initialize(storage_config):
-    passphrase = storage_config.get('encryption_passphrase')
-
-    if passphrase:
-        os.environ['BORG_PASSPHRASE'] = passphrase
-
-
-def _write_exclude_file(exclude_patterns=None):
+def _expand_directory(directory):
     '''
-    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
-    None if no patterns are provided.
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
     '''
-    if not exclude_patterns:
+    expanded_directory = os.path.expanduser(directory)
+
+    return glob.glob(expanded_directory) or [expanded_directory]
+
+
+def _expand_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(_expand_directory(directory) for directory in directories)
+    )
+
+
+def _expand_home_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
+    Return the results as a tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(os.path.expanduser(directory) for directory in directories)
+
+
+def _write_pattern_file(patterns=None):
+    '''
+    Given a sequence of patterns, write them to a named temporary file and return it. Return None
+    if no patterns are provided.
+    '''
+    if not patterns:
         return None
 
-    exclude_file = tempfile.NamedTemporaryFile('w')
-    exclude_file.write('\n'.join(exclude_patterns))
-    exclude_file.flush()
+    pattern_file = tempfile.NamedTemporaryFile('w')
+    pattern_file.write('\n'.join(patterns))
+    pattern_file.flush()
 
-    return exclude_file
+    return pattern_file
 
 
-def _make_exclude_flags(location_config, exclude_patterns_filename=None):
+def _make_pattern_flags(location_config, pattern_filename=None):
+    '''
+    Given a location config dict with a potential pattern_from option, and a filename containing any
+    additional patterns, return the corresponding Borg flags for those files as a tuple.
+    '''
+    pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
+        (pattern_filename,) if pattern_filename else ()
+    )
+
+    return tuple(
+        itertools.chain.from_iterable(
+            ('--patterns-from', pattern_filename) for pattern_filename in pattern_filenames
+        )
+    )
+
+
+def _make_exclude_flags(location_config, exclude_filename=None):
     '''
     Given a location config dict with various exclude options, and a filename containing any exclude
     patterns, return the corresponding Borg flags as a tuple.
     '''
     exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
-        (exclude_patterns_filename,) if exclude_patterns_filename else ()
+        (exclude_filename,) if exclude_filename else ()
     )
     exclude_from_flags = tuple(
         itertools.chain.from_iterable(
-            ('--exclude-from', exclude_filename)
-            for exclude_filename in exclude_filenames
+            ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
         )
     )
     caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
@@ -51,45 +95,77 @@ def _make_exclude_flags(location_config, exclude_patterns_filename=None):
 
 
 def create_archive(
-    verbosity, repository, location_config, storage_config,
+    dry_run,
+    repository,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+    stats=False,
+    json=False,
 ):
     '''
-    Given a vebosity flag, a local or remote repository path, a location config dict, and a storage
-    config dict, create a Borg archive.
+    Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
+    storage config dict, create a Borg archive and return Borg's JSON output (if any).
     '''
-    sources = tuple(
-        itertools.chain.from_iterable(
-            glob.glob(directory) or [directory]
-            for directory in location_config['source_directories']
-        )
-    )
+    sources = _expand_directories(location_config['source_directories'])
 
-    exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns'))
-    exclude_flags = _make_exclude_flags(
-        location_config,
-        exclude_patterns_file.name if exclude_patterns_file else None,
+    pattern_file = _write_pattern_file(location_config.get('patterns'))
+    exclude_file = _write_pattern_file(
+        _expand_home_directories(location_config.get('exclude_patterns'))
     )
+    checkpoint_interval = storage_config.get('checkpoint_interval', None)
+    chunker_params = storage_config.get('chunker_params', None)
     compression = storage_config.get('compression', None)
-    compression_flags = ('--compression', compression) if compression else ()
+    remote_rate_limit = storage_config.get('remote_rate_limit', None)
     umask = storage_config.get('umask', None)
-    umask_flags = ('--umask', str(umask)) if umask else ()
-    one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
-    remote_path = location_config.get('remote_path')
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
-    }.get(verbosity, ())
+    lock_wait = storage_config.get('lock_wait', None)
+    files_cache = location_config.get('files_cache')
     default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
     archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
 
     full_command = (
-        'borg', 'create',
-        '{repository}::{archive_name_format}'.format(
-            repository=repository,
-            archive_name_format=archive_name_format,
-        ),
-    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
-        remote_path_flags + umask_flags + verbosity_flags
+        (
+            local_path,
+            'create',
+            '{repository}::{archive_name_format}'.format(
+                repository=repository, archive_name_format=archive_name_format
+            ),
+        )
+        + sources
+        + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
+        + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
+        + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
+        + (('--compression', compression) if compression else ())
+        + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
+        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--read-special',) if location_config.get('read_special') else ())
+        + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
+        + (('--files-cache', files_cache) if files_cache else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--list', '--filter', 'AME-') if logger.isEnabledFor(logging.INFO) and not json else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (
+            ('--stats',)
+            if not dry_run and (logger.isEnabledFor(logging.INFO) or stats) and not json
+            else ()
+        )
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+        + (('--json',) if json else ())
+    )
 
-    subprocess.check_call(full_command)
+    if json:
+        output_log_level = None
+    elif stats:
+        output_log_level = logging.WARNING
+    else:
+        output_log_level = logging.INFO
+
+    return execute_command(full_command, output_log_level)

borgmatic/borg/environment.py

@@ -0,0 +1,19 @@
+import os
+
+OPTION_TO_ENVIRONMENT_VARIABLE = {
+    'borg_base_directory': 'BORG_BASE_DIR',
+    'borg_config_directory': 'BORG_CONFIG_DIR',
+    'borg_cache_directory': 'BORG_CACHE_DIR',
+    'borg_security_directory': 'BORG_SECURITY_DIR',
+    'borg_keys_directory': 'BORG_KEYS_DIR',
+    'encryption_passcommand': 'BORG_PASSCOMMAND',
+    'encryption_passphrase': 'BORG_PASSPHRASE',
+    'ssh_command': 'BORG_RSH',
+}
+
+
+def initialize(storage_config):
+    for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
+        value = storage_config.get(option_name)
+        if value:
+            os.environ[environment_variable_name] = value

borgmatic/borg/extract.py

@@ -1,40 +1,86 @@
-import sys
-import subprocess
+import logging
 
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
 
 
-def extract_last_archive_dry_run(verbosity, repository, remote_path=None):
+def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg', remote_path=None):
     '''
-    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
-    the dry-run.
+    Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
+    dry-run.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info',),
-        VERBOSITY_LOTS: ('--debug',),
-    }.get(verbosity, ())
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
+    verbosity_flags = ()
+    if logger.isEnabledFor(logging.DEBUG):
+        verbosity_flags = ('--debug', '--show-rc')
+    elif logger.isEnabledFor(logging.INFO):
+        verbosity_flags = ('--info',)
 
     full_list_command = (
-        'borg', 'list',
-        '--short',
-        repository,
-    ) + remote_path_flags + verbosity_flags
+        (local_path, 'list', '--short', repository)
+        + remote_path_flags
+        + lock_wait_flags
+        + verbosity_flags
+    )
 
-    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
+    list_output = execute_command(full_list_command, output_log_level=None)
 
-    last_archive_name = list_output.strip().split('\n')[-1]
-    if not last_archive_name:
+    try:
+        last_archive_name = list_output.strip().splitlines()[-1]
+    except IndexError:
         return
 
-    list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
+    list_flag = ('--list',) if logger.isEnabledFor(logging.DEBUG) else ()
     full_extract_command = (
-        'borg', 'extract',
-        '--dry-run',
-        '{repository}::{last_archive_name}'.format(
-            repository=repository,
-            last_archive_name=last_archive_name,
-        ),
-    ) + remote_path_flags + verbosity_flags + list_flag
+        (
+            local_path,
+            'extract',
+            '--dry-run',
+            '{repository}::{last_archive_name}'.format(
+                repository=repository, last_archive_name=last_archive_name
+            ),
+        )
+        + remote_path_flags
+        + lock_wait_flags
+        + verbosity_flags
+        + list_flag
+    )
 
-    subprocess.check_call(full_extract_command)
+    execute_command(full_extract_command)
+
+
+def extract_archive(
+    dry_run,
+    repository,
+    archive,
+    restore_paths,
+    location_config,
+    storage_config,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
+    restore from the archive, and location/storage configuration dicts, extract the archive into the
+    current directory.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'extract', '::'.join((repository, archive)))
+        + (tuple(restore_paths) if restore_paths else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
+    )
+
+    execute_command(full_command)

borgmatic/borg/info.py

@@ -0,0 +1,26 @@
+import logging
+
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def display_archives_info(
+    repository, storage_config, local_path='borg', remote_path=None, json=False
+):
+    '''
+    Given a local or remote repository path, and a storage config dict, display summary information
+    for Borg archives in the repository or return JSON summary information.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'info', repository)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
+        + (('--json',) if json else ())
+    )
+
+    return execute_command(full_command, output_log_level=None if json else logging.WARNING)

borgmatic/borg/init.py

@@ -0,0 +1,47 @@
+import logging
+import subprocess
+
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
+
+
+def initialize_repository(
+    repository,
+    encryption_mode,
+    append_only=None,
+    storage_quota=None,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository path, a Borg encryption mode, whether the repository should
+    be append-only, and the storage quota to use, initialize the repository. If the repository
+    already exists, then log and skip initialization.
+    '''
+    info_command = (local_path, 'info', repository)
+    logger.debug(' '.join(info_command))
+
+    try:
+        execute_command(info_command, output_log_level=None)
+        logger.info('Repository already exists. Skipping initialization.')
+        return
+    except subprocess.CalledProcessError as error:
+        if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
+            raise
+
+    init_command = (
+        (local_path, 'init', repository)
+        + (('--encryption', encryption_mode) if encryption_mode else ())
+        + (('--append-only',) if append_only else ())
+        + (('--storage-quota', storage_quota) if storage_quota else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+    )
+
+    # Don't use execute_command() here because it doesn't support interactive prompts.
+    subprocess.check_call(init_command)

borgmatic/borg/list.py

@@ -0,0 +1,27 @@
+import logging
+
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def list_archives(
+    repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
+):
+    '''
+    Given a local or remote repository path and a storage config dict, display the output of listing
+    Borg archives in the repository or return JSON output. Or, if an archive name is given, listing
+    the files in that archive.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
+        + (('--json',) if json else ())
+    )
+
+    return execute_command(full_command, output_log_level=None if json else logging.WARNING)

borgmatic/borg/prune.py

@@ -1,6 +1,8 @@
-import subprocess
+import logging
 
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
 
 
 def _make_prune_flags(retention_config):
@@ -19,30 +21,43 @@ def _make_prune_flags(retention_config):
             ('--keep-monthly', '6'),
         )
     '''
+    if not retention_config.get('prefix'):
+        retention_config['prefix'] = '{hostname}-'
+
     return (
         ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
         for option_name, value in retention_config.items()
     )
 
 
-def prune_archives(verbosity, repository, retention_config, remote_path=None):
+def prune_archives(
+    dry_run,
+    repository,
+    storage_config,
+    retention_config,
+    local_path='borg',
+    remote_path=None,
+    stats=False,
+):
     '''
-    Given a verbosity flag, a local or remote repository path, a retention config dict, prune Borg
-    archives according the the retention policy specified in that configuration.
+    Given dry-run flag, a local or remote repository path, a storage config dict, and a
+    retention config dict, prune Borg archives according to the retention policy specified in that
+    configuration.
     '''
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
-    }.get(verbosity, ())
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        'borg', 'prune',
-        repository,
-    ) + tuple(
-        element
-        for pair in _make_prune_flags(retention_config)
-        for element in pair
-    ) + remote_path_flags + verbosity_flags
+        (local_path, 'prune', repository)
+        + tuple(element for pair in _make_prune_flags(retention_config) for element in pair)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (('--stats',) if stats else ())
+    )
 
-    subprocess.check_call(full_command)
+    execute_command(full_command)

borgmatic/commands/borgmatic.py

@@ -1,111 +1,549 @@
-from __future__ import print_function
-from argparse import ArgumentParser
+import collections
+import json
+import logging
 import os
-from subprocess import CalledProcessError
 import sys
+from argparse import ArgumentParser
+from subprocess import CalledProcessError
 
-from borgmatic.borg import check, create, prune
-from borgmatic.config import collect, convert, validate
+import colorama
+import pkg_resources
 
+from borgmatic import hook
+from borgmatic.borg import check as borg_check
+from borgmatic.borg import create as borg_create
+from borgmatic.borg import environment as borg_environment
+from borgmatic.borg import extract as borg_extract
+from borgmatic.borg import info as borg_info
+from borgmatic.borg import init as borg_init
+from borgmatic.borg import list as borg_list
+from borgmatic.borg import prune as borg_prune
+from borgmatic.config import checks, collect, convert, validate
+from borgmatic.logger import configure_logging, should_do_markup
+from borgmatic.signals import configure_signals
+from borgmatic.verbosity import verbosity_to_log_level
+
+logger = logging.getLogger(__name__)
 
 LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
-DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
-DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes'
 
 
 def parse_arguments(*arguments):
     '''
     Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
+    them as an argparse.ArgumentParser instance.
     '''
+    config_paths = collect.get_default_config_paths()
+
     parser = ArgumentParser(
-        description=
-            '''
+        description='''
             A simple wrapper script for the Borg backup software that creates and prunes backups.
-            If none of the --prune, --create, or --check options are given, then borgmatic defaults
-            to all three: prune, create, and check archives.
-            '''
+            If none of the action options are given, then borgmatic defaults to: prune, create, and
+            check archives.
+            ''',
+        add_help=False,
     )
-    parser.add_argument(
-        '-c', '--config',
-        nargs='+',
-        dest='config_paths',
-        default=DEFAULT_CONFIG_PATHS,
-        help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)),
+
+    actions_group = parser.add_argument_group('actions')
+    actions_group.add_argument(
+        '-I', '--init', dest='init', action='store_true', help='Initialize an empty Borg repository'
     )
-    parser.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        help='Deprecated in favor of exclude_patterns within configuration',
-    )
-    parser.add_argument(
-        '-p', '--prune',
+    actions_group.add_argument(
+        '-p',
+        '--prune',
         dest='prune',
         action='store_true',
         help='Prune archives according to the retention policy',
     )
-    parser.add_argument(
-        '-C', '--create',
+    actions_group.add_argument(
+        '-C',
+        '--create',
         dest='create',
         action='store_true',
         help='Create archives (actually perform backups)',
     )
-    parser.add_argument(
-        '-k', '--check',
-        dest='check',
+    actions_group.add_argument(
+        '-k', '--check', dest='check', action='store_true', help='Check archives for consistency'
+    )
+
+    actions_group.add_argument(
+        '-x',
+        '--extract',
+        dest='extract',
         action='store_true',
-        help='Check archives for consistency',
+        help='Extract a named archive to the current directory',
     )
-    parser.add_argument(
-        '-v', '--verbosity',
+    actions_group.add_argument(
+        '-l', '--list', dest='list', action='store_true', help='List archives'
+    )
+    actions_group.add_argument(
+        '-i',
+        '--info',
+        dest='info',
+        action='store_true',
+        help='Display summary information on archives',
+    )
+
+    init_group = parser.add_argument_group('options for --init')
+    init_group.add_argument(
+        '-e', '--encryption', dest='encryption_mode', help='Borg repository encryption mode'
+    )
+    init_group.add_argument(
+        '--append-only',
+        dest='append_only',
+        action='store_true',
+        help='Create an append-only repository',
+    )
+    init_group.add_argument(
+        '--storage-quota',
+        dest='storage_quota',
+        help='Create a repository with a fixed storage quota',
+    )
+
+    prune_group = parser.add_argument_group('options for --prune')
+    stats_argument = prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+
+    create_group = parser.add_argument_group('options for --create')
+    progress_argument = create_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is processed',
+    )
+    create_group._group_actions.append(stats_argument)
+    json_argument = create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+    )
+
+    extract_group = parser.add_argument_group('options for --extract')
+    repository_argument = extract_group.add_argument(
+        '--repository',
+        help='Path of repository to use, defaults to the configured repository if there is only one',
+    )
+    archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
+    extract_group.add_argument(
+        '--restore-path',
+        nargs='+',
+        dest='restore_paths',
+        help='Paths to restore from archive, defaults to the entire archive',
+    )
+    extract_group._group_actions.append(progress_argument)
+
+    list_group = parser.add_argument_group('options for --list')
+    list_group._group_actions.append(repository_argument)
+    list_group._group_actions.append(archive_argument)
+    list_group._group_actions.append(json_argument)
+
+    info_group = parser.add_argument_group('options for --info')
+    info_group._group_actions.append(json_argument)
+
+    common_group = parser.add_argument_group('common options')
+    common_group.add_argument(
+        '-c',
+        '--config',
+        nargs='+',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+    common_group.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    common_group.add_argument(
+        '-n',
+        '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write to any repositories',
+    )
+    common_group.add_argument(
+        '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output'
+    )
+    common_group.add_argument(
+        '-v',
+        '--verbosity',
         type=int,
-        help='Display verbose progress (1 for some, 2 for lots)',
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to the console (from none to lots: 0, 1, or 2)',
     )
+    common_group.add_argument(
+        '--syslog-verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=0,
+        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2)',
+    )
+    common_group.add_argument(
+        '--version',
+        dest='version',
+        default=False,
+        action='store_true',
+        help='Display installed version number of borgmatic and exit',
+    )
+    common_group.add_argument('--help', action='help', help='Show this help information and exit')
 
     args = parser.parse_args(arguments)
 
-    # If any of the three action flags in the given parse arguments have been explicitly requested,
-    # leave them as-is. Otherwise, assume defaults: Mutate the given arguments to enable all the
-    # actions.
-    if not args.prune and not args.create and not args.check:
+    if args.excludes_filename:
+        raise ValueError(
+            'The --excludes option has been replaced with exclude_patterns in configuration'
+        )
+
+    if (args.encryption_mode or args.append_only or args.storage_quota) and not args.init:
+        raise ValueError(
+            'The --encryption, --append-only, and --storage-quota options can only be used with the --init option'
+        )
+
+    if args.init and args.dry_run:
+        raise ValueError('The --init option cannot be used with the --dry-run option')
+    if args.init and not args.encryption_mode:
+        raise ValueError('The --encryption option is required with the --init option')
+
+    if not args.extract:
+        if not args.list:
+            if args.repository:
+                raise ValueError(
+                    'The --repository option can only be used with the --extract and --list options'
+                )
+            if args.archive:
+                raise ValueError(
+                    'The --archive option can only be used with the --extract and --list options'
+                )
+        if args.restore_paths:
+            raise ValueError('The --restore-path option can only be used with the --extract option')
+    if args.extract and not args.archive:
+        raise ValueError('The --archive option is required with the --extract option')
+
+    if args.progress and not (args.create or args.extract):
+        raise ValueError(
+            'The --progress option can only be used with the --create and --extract options'
+        )
+
+    if args.json and not (args.create or args.list or args.info):
+        raise ValueError(
+            'The --json option can only be used with the --create, --list, or --info options'
+        )
+
+    if args.json and args.list and args.info:
+        raise ValueError(
+            'With the --json option, options --list and --info cannot be used together'
+        )
+
+    # If any of the action flags are explicitly requested, leave them as-is. Otherwise, assume
+    # defaults: Mutate the given arguments to enable the default actions.
+    if (
+        not args.init
+        and not args.prune
+        and not args.create
+        and not args.check
+        and not args.extract
+        and not args.list
+        and not args.info
+    ):
         args.prune = True
         args.create = True
         args.check = True
 
+    if args.stats and not (args.create or args.prune):
+        raise ValueError('The --stats option can only be used when creating or pruning archives')
+
     return args
 
 
+def run_configuration(config_filename, config, args):  # pragma: no cover
+    '''
+    Given a config filename and the corresponding parsed config dict, execute its defined pruning,
+    backups, consistency checks, and/or other actions.
+
+    Yield JSON output strings from executing any actions that produce JSON.
+    '''
+    (location, storage, retention, consistency, hooks) = (
+        config.get(section_name, {})
+        for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
+    )
+
+    try:
+        local_path = location.get('local_path', 'borg')
+        remote_path = location.get('remote_path')
+        borg_environment.initialize(storage)
+
+        if args.create:
+            hook.execute_hook(
+                hooks.get('before_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'pre-backup',
+                args.dry_run,
+            )
+
+        for repository_path in location['repositories']:
+            yield from run_actions(
+                args=args,
+                location=location,
+                storage=storage,
+                retention=retention,
+                consistency=consistency,
+                local_path=local_path,
+                remote_path=remote_path,
+                repository_path=repository_path,
+            )
+
+        if args.create:
+            hook.execute_hook(
+                hooks.get('after_backup'),
+                hooks.get('umask'),
+                config_filename,
+                'post-backup',
+                args.dry_run,
+            )
+    except (OSError, CalledProcessError):
+        hook.execute_hook(
+            hooks.get('on_error'), hooks.get('umask'), config_filename, 'on-error', args.dry_run
+        )
+        raise
+
+
+def run_actions(
+    *, args, location, storage, retention, consistency, local_path, remote_path, repository_path
+):  # pragma: no cover
+    '''
+    Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
+    configuration dicts, local and remote paths to Borg, and a repository name, run all actions
+    from the command-line arguments on the given repository.
+
+    Yield JSON output strings from executing any actions that produce JSON.
+    '''
+    repository = os.path.expanduser(repository_path)
+    dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
+    if args.init:
+        logger.info('{}: Initializing repository'.format(repository))
+        borg_init.initialize_repository(
+            repository,
+            args.encryption_mode,
+            args.append_only,
+            args.storage_quota,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+    if args.prune:
+        logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
+        borg_prune.prune_archives(
+            args.dry_run,
+            repository,
+            storage,
+            retention,
+            local_path=local_path,
+            remote_path=remote_path,
+            stats=args.stats,
+        )
+    if args.create:
+        logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        json_output = borg_create.create_archive(
+            args.dry_run,
+            repository,
+            location,
+            storage,
+            local_path=local_path,
+            remote_path=remote_path,
+            progress=args.progress,
+            stats=args.stats,
+            json=args.json,
+        )
+        if json_output:
+            yield json.loads(json_output)
+    if args.check and checks.repository_enabled_for_checks(repository, consistency):
+        logger.info('{}: Running consistency checks'.format(repository))
+        borg_check.check_archives(
+            repository, storage, consistency, local_path=local_path, remote_path=remote_path
+        )
+    if args.extract:
+        if args.repository is None or repository == args.repository:
+            logger.info('{}: Extracting archive {}'.format(repository, args.archive))
+            borg_extract.extract_archive(
+                args.dry_run,
+                repository,
+                args.archive,
+                args.restore_paths,
+                location,
+                storage,
+                local_path=local_path,
+                remote_path=remote_path,
+                progress=args.progress,
+            )
+    if args.list:
+        if args.repository is None or repository == args.repository:
+            logger.info('{}: Listing archives'.format(repository))
+            json_output = borg_list.list_archives(
+                repository,
+                storage,
+                args.archive,
+                local_path=local_path,
+                remote_path=remote_path,
+                json=args.json,
+            )
+            if json_output:
+                yield json.loads(json_output)
+    if args.info:
+        logger.info('{}: Displaying summary info for archives'.format(repository))
+        json_output = borg_info.display_archives_info(
+            repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
+        )
+        if json_output:
+            yield json.loads(json_output)
+
+
+def load_configurations(config_filenames):
+    '''
+    Given a sequence of configuration filenames, load and validate each configuration file. Return
+    the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
+    and sequence of logging.LogRecord instances containing any parse errors.
+    '''
+    # Dict mapping from config filename to corresponding parsed config dict.
+    configs = collections.OrderedDict()
+    logs = []
+
+    # Parse and load each configuration file.
+    for config_filename in config_filenames:
+        try:
+            configs[config_filename] = validate.parse_configuration(
+                config_filename, validate.schema_filename()
+            )
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logs.extend(
+                [
+                    logging.makeLogRecord(
+                        dict(
+                            levelno=logging.CRITICAL,
+                            levelname='CRITICAL',
+                            msg='{}: Error parsing configuration file'.format(config_filename),
+                        )
+                    ),
+                    logging.makeLogRecord(
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+                    ),
+                ]
+            )
+
+    return (configs, logs)
+
+
+def collect_configuration_run_summary_logs(configs, args):
+    '''
+    Given a dict of configuration filename to corresponding parsed configuration, and parsed
+    command-line arguments as an argparse.ArgumentParser instance, run each configuration file and
+    yield a series of logging.LogRecord instances containing summary information about each run.
+
+    As a side effect of running through these configuration files, output their JSON results, if
+    any, to stdout.
+    '''
+    # Run cross-file validation checks.
+    if args.extract or (args.list and args.archive):
+        try:
+            validate.guard_configuration_contains_repository(args.repository, configs)
+        except ValueError as error:
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            )
+            return
+
+    # Execute the actions corresponding to each configuration file.
+    json_results = []
+    for config_filename, config in configs.items():
+        try:
+            json_results.extend(list(run_configuration(config_filename, config, args)))
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.INFO,
+                    levelname='INFO',
+                    msg='{}: Successfully ran configuration file'.format(config_filename),
+                )
+            )
+        except (ValueError, OSError, CalledProcessError) as error:
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.CRITICAL,
+                    levelname='CRITICAL',
+                    msg='{}: Error running configuration file'.format(config_filename),
+                )
+            )
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            )
+
+    if json_results:
+        sys.stdout.write(json.dumps(json_results))
+
+    if not configs:
+        yield logging.makeLogRecord(
+            dict(
+                levelno=logging.CRITICAL,
+                levelname='CRITICAL',
+                msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
+            )
+        )
+
+
+def exit_with_help_link():  # pragma: no cover
+    '''
+    Display a link to get help and exit with an error code.
+    '''
+    logger.critical('')
+    logger.critical('Need some help? https://torsion.org/borgmatic/#issues')
+    sys.exit(1)
+
+
 def main():  # pragma: no cover
+    configure_signals()
+
     try:
         args = parse_arguments(*sys.argv[1:])
-        config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
-        convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
+    except ValueError as error:
+        configure_logging(logging.CRITICAL)
+        logger.critical(error)
+        exit_with_help_link()
+    except SystemExit as error:
+        if error.code == 0:
+            raise error
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error parsing arguments: {}'.format(' '.join(sys.argv)))
+        exit_with_help_link()
 
-        if len(config_filenames) == 0:
-            raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths)))
+    if args.version:
+        print(pkg_resources.require('borgmatic')[0].version)
+        sys.exit(0)
 
-        for config_filename in config_filenames:
-            config = validate.parse_configuration(config_filename, validate.schema_filename())
-            (location, storage, retention, consistency) = (
-                config.get(section_name, {})
-                for section_name in ('location', 'storage', 'retention', 'consistency')
-            )
-            remote_path = location.get('remote_path')
+    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    configs, parse_logs = load_configurations(config_filenames)
 
-            create.initialize(storage)
+    colorama.init(autoreset=True, strip=not should_do_markup(args.no_color, configs))
+    configure_logging(
+        verbosity_to_log_level(args.verbosity), verbosity_to_log_level(args.syslog_verbosity)
+    )
 
-            for repository in location['repositories']:
-                if args.prune:
-                    prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
-                if args.create:
-                    create.create_archive(
-                        args.verbosity,
-                        repository,
-                        location,
-                        storage,
-                    )
-                if args.check:
-                    check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
-    except (ValueError, OSError, CalledProcessError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)
+    logger.debug('Ensuring legacy configuration is upgraded')
+    convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
+
+    summary_logs = list(collect_configuration_run_summary_logs(configs, args))
+
+    logger.info('')
+    logger.info('summary:')
+    [
+        logger.handle(log)
+        for log in parse_logs + summary_logs
+        if log.levelno >= logger.getEffectiveLevel()
+    ]
+
+    if any(log.levelno == logging.CRITICAL for log in summary_logs):
+        exit_with_help_link()

borgmatic/commands/convert_config.py

@@ -1,15 +1,12 @@
-from __future__ import print_function
-from argparse import ArgumentParser
 import os
-from subprocess import CalledProcessError
 import sys
 import textwrap
+from argparse import ArgumentParser
 
 from ruamel import yaml
 
 from borgmatic.config import convert, generate, legacy, validate
 
-
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@@ -27,22 +24,31 @@ def parse_arguments(*arguments):
         '''
     )
     parser.add_argument(
-        '-s', '--source-config',
+        '-s',
+        '--source-config',
         dest='source_config_filename',
         default=DEFAULT_SOURCE_CONFIG_FILENAME,
-        help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
+        help='Source INI-style configuration filename. Default: {}'.format(
+            DEFAULT_SOURCE_CONFIG_FILENAME
+        ),
     )
     parser.add_argument(
-        '-e', '--source-excludes',
+        '-e',
+        '--source-excludes',
         dest='source_excludes_filename',
-        default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None,
+        default=DEFAULT_SOURCE_EXCLUDES_FILENAME
+        if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME)
+        else None,
         help='Excludes filename',
     )
     parser.add_argument(
-        '-d', '--destination-config',
+        '-d',
+        '--destination-config',
         dest='destination_config_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+        help='Destination YAML configuration filename. Default: {}'.format(
+            DEFAULT_DESTINATION_CONFIG_FILENAME
+        ),
     )
 
     return parser.parse_args(arguments)
@@ -58,11 +64,13 @@ def display_result(args):  # pragma: no cover
         ),
         TEXT_WRAP_CHARACTERS,
     )
-    
+
     delete_lines = textwrap.wrap(
         'Once you are satisfied, you can safely delete {}{}.'.format(
             args.source_config_filename,
-            ' and {}'.format(args.source_excludes_filename) if args.source_excludes_filename else '',
+            ' and {}'.format(args.source_excludes_filename)
+            if args.source_excludes_filename
+            else '',
         ),
         TEXT_WRAP_CHARACTERS,
     )
@@ -76,7 +84,9 @@ def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
         schema = yaml.round_trip_load(open(validate.schema_filename()).read())
-        source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
+        source_config = legacy.parse_configuration(
+            args.source_config_filename, legacy.CONFIG_FORMAT
+        )
         source_config_file_mode = os.stat(args.source_config_filename).st_mode
         source_excludes = (
             open(args.source_excludes_filename).read().splitlines()
@@ -84,12 +94,12 @@ def main():  # pragma: no cover
             else []
         )
 
-        destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
+        destination_config = convert.convert_legacy_parsed_config(
+            source_config, source_excludes, schema
+        )
 
         generate.write_configuration(
-            args.destination_config_filename,
-            destination_config,
-            mode=source_config_file_mode,
+            args.destination_config_filename, destination_config, mode=source_config_file_mode
         )
 
         display_result(args)

borgmatic/commands/generate_config.py

@@ -1,11 +1,7 @@
-from __future__ import print_function
-from argparse import ArgumentParser
-import os
-from subprocess import CalledProcessError
 import sys
+from argparse import ArgumentParser
 
-from borgmatic.config import convert, generate, validate
-
+from borgmatic.config import generate, validate
 
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
@@ -17,10 +13,13 @@ def parse_arguments(*arguments):
     '''
     parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
     parser.add_argument(
-        '-d', '--destination',
+        '-d',
+        '--destination',
         dest='destination_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+        help='Destination YAML configuration filename. Default: {}'.format(
+            DEFAULT_DESTINATION_CONFIG_FILENAME
+        ),
     )
 
     return parser.parse_args(arguments)
@@ -30,12 +29,16 @@ def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
 
-        generate.generate_sample_configuration(args.destination_filename, validate.schema_filename())
+        generate.generate_sample_configuration(
+            args.destination_filename, validate.schema_filename()
+        )
 
         print('Generated a sample configuration file at {}.'.format(args.destination_filename))
         print()
-        print('Please edit the file to suit your needs. The values are just representative.')
+        print('Please edit the file to suit your needs. The values are representative.')
         print('All fields are optional except where indicated.')
+        print()
+        print('If you ever need help: https://torsion.org/borgmatic/#issues')
     except (ValueError, OSError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

borgmatic/commands/validate_config.py

@@ -0,0 +1,56 @@
+import logging
+import sys
+from argparse import ArgumentParser
+
+from borgmatic.config import collect, validate
+
+logger = logging.getLogger(__name__)
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
+    parser.add_argument(
+        '-c',
+        '--config',
+        nargs='+',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(
+            ' '.join(config_paths)
+        ),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():  # pragma: no cover
+    args = parse_arguments(*sys.argv[1:])
+
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+    if len(config_filenames) == 0:
+        logger.critical('No files to validate found')
+        sys.exit(1)
+
+    found_issues = False
+    for config_filename in config_filenames:
+        try:
+            validate.parse_configuration(config_filename, validate.schema_filename())
+        except (ValueError, OSError, validate.Validation_error) as error:
+            logging.critical('{}: Error parsing configuration file'.format(config_filename))
+            logging.critical(error)
+            found_issues = True
+
+    if found_issues:
+        sys.exit(1)
+    else:
+        logger.info(
+            'All given configuration files are valid: {}'.format(', '.join(config_filenames))
+        )

borgmatic/config/checks.py

@@ -0,0 +1,9 @@
+def repository_enabled_for_checks(repository, consistency):
+    '''
+    Given a repository name and a consistency configuration dict, return whether the repository
+    is enabled to have consistency checks run.
+    '''
+    if not consistency.get('check_repositories'):
+        return True
+
+    return repository in consistency['check_repositories']

borgmatic/config/collect.py

@@ -1,27 +1,48 @@
 import os
 
 
+def get_default_config_paths():
+    '''
+    Based on the value of the XDG_CONFIG_HOME and HOME environment variables, return a list of
+    default configuration paths. This includes both system-wide configuration and configuration in
+    the current user's home directory.
+    '''
+    user_config_directory = os.getenv('XDG_CONFIG_HOME') or os.path.expandvars(
+        os.path.join('$HOME', '.config')
+    )
+
+    return [
+        '/etc/borgmatic/config.yaml',
+        '/etc/borgmatic.d',
+        '%s/borgmatic/config.yaml' % user_config_directory,
+    ]
+
+
 def collect_config_filenames(config_paths):
     '''
-    Given a sequence of config paths, both filenames and directories, resolve that to just an
-    iterable of files. Accomplish this by listing any given directories looking for contained config
-    files. This is non-recursive, so any directories within the given directories are ignored.
+    Given a sequence of config paths, both filenames and directories, resolve that to an iterable
+    of files. Accomplish this by listing any given directories looking for contained config files
+    (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given
+    directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
-    configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to
-    create it unless they need it.
+    configuration paths. However, skip a default config path if it's missing, so the user doesn't
+    have to create a default config path unless they need it.
     '''
+    real_default_config_paths = set(map(os.path.realpath, get_default_config_paths()))
+
     for path in config_paths:
         exists = os.path.exists(path)
 
-        if os.path.realpath(path) == '/etc/borgmatic.d' and not exists:
+        if os.path.realpath(path) in real_default_config_paths and not exists:
             continue
 
         if not os.path.isdir(path) or not exists:
             yield path
             continue
 
-        for filename in os.listdir(path):
+        for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
-            if not os.path.isdir(full_filename):
+            matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
+            if matching_filetype and not os.path.isdir(full_filename):
                 yield full_filename

borgmatic/config/convert.py

@@ -12,14 +12,17 @@ def _convert_section(source_section_config, section_schema):
 
     Where integer types exist in the given section schema, convert their values to integers.
     '''
-    destination_section_config = yaml.comments.CommentedMap([
-        (
-            option_name,
-            int(option_value)
-                if section_schema['map'].get(option_name, {}).get('type') == 'int' else option_value
-        )
-        for option_name, option_value in source_section_config.items()
-    ])
+    destination_section_config = yaml.comments.CommentedMap(
+        [
+            (
+                option_name,
+                int(option_value)
+                if section_schema['map'].get(option_name, {}).get('type') == 'int'
+                else option_value,
+            )
+            for option_name, option_value in source_section_config.items()
+        ]
+    )
 
     return destination_section_config
 
@@ -33,10 +36,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
     Additionally, use the given schema as a source of helpful comments to include within the
     returned CommentedMap.
     '''
-    destination_config = yaml.comments.CommentedMap([
-        (section_name, _convert_section(section_config, schema['map'][section_name]))
-        for section_name, section_config in source_config._asdict().items()
-    ])
+    destination_config = yaml.comments.CommentedMap(
+        [
+            (section_name, _convert_section(section_config, schema['map'][section_name]))
+            for section_name, section_config in source_config._asdict().items()
+        ]
+    )
 
     # Split space-seperated values into actual lists, make "repository" into a list, and merge in
     # excludes.
@@ -53,17 +58,15 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
 
     for section_name, section_config in destination_config.items():
         generate.add_comments_to_configuration(
-            section_config,
-            schema['map'][section_name],
-            indent=generate.INDENT,
+            section_config, schema['map'][section_name], indent=generate.INDENT
         )
 
     return destination_config
 
 
-class LegacyConfigurationNotUpgraded(FileNotFoundError):
+class Legacy_configuration_not_upgraded(FileNotFoundError):
     def __init__(self):
-        super(LegacyConfigurationNotUpgraded, self).__init__(
+        super(Legacy_configuration_not_upgraded, self).__init__(
             '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
 to YAML. This better supports validation, and has a more natural way to express
 lists of values. To upgrade your existing configuration, run:
@@ -80,32 +83,13 @@ instead of the old one.'''
 def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
     '''
     If legacy source configuration exists but no destination upgraded configs do, raise
-    LegacyConfigurationNotUpgraded.
+    Legacy_configuration_not_upgraded.
 
     The idea is that we want to alert the user about upgrading their config if they haven't already.
     '''
     destination_config_exists = any(
-        os.path.exists(filename)
-        for filename in destination_config_filenames
+        os.path.exists(filename) for filename in destination_config_filenames
     )
 
     if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise LegacyConfigurationNotUpgraded()
-
-
-class LegacyExcludesFilenamePresent(FileNotFoundError):
-    def __init__(self):
-        super(LegacyExcludesFilenamePresent, self).__init__(
-            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
-to YAML. This better supports validation, and has a more natural way to express
-lists of values. The new configuration file incorporates excludes, so you no
-longer need to provide an excludes filename on the command-line with an
-"--excludes" argument.
-
-Please remove the "--excludes" argument and run borgmatic again.'''
-        )
-
-
-def guard_excludes_filename_omitted(excludes_filename):
-    if excludes_filename != None:
-        raise LegacyExcludesFilenamePresent()
+        raise Legacy_configuration_not_upgraded()

borgmatic/config/generate.py

@@ -1,20 +1,17 @@
-from collections import OrderedDict
 import os
 
 from ruamel import yaml
 
-
 INDENT = 4
 
 
 def _insert_newline_before_comment(config, field_name):
     '''
-    Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
+    Using some ruamel.yaml black magic, insert a blank line in the config right before the given
     field and its comments.
     '''
     config.ca.items[field_name][1].insert(
-        0,
-        yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
+        0, yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None)
     )
 
 
@@ -24,26 +21,82 @@ def _schema_to_sample_configuration(schema, level=0):
     for each section based on the schema "desc" description.
     '''
     example = schema.get('example')
-    if example:
+    if example is not None:
         return example
 
-    config = yaml.comments.CommentedMap([
-        (
-            section_name,
-            _schema_to_sample_configuration(section_schema, level + 1),
-        )
-        for section_name, section_schema in schema['map'].items()
-    ])
+    config = yaml.comments.CommentedMap(
+        [
+            (section_name, _schema_to_sample_configuration(section_schema, level + 1))
+            for section_name, section_schema in schema['map'].items()
+        ]
+    )
 
     add_comments_to_configuration(config, schema, indent=(level * INDENT))
 
     return config
 
 
-def write_configuration(config_filename, config, mode=0o600):
+def _comment_out_line(line):
+    # If it's already is commented out (or empty), there's nothing further to do!
+    stripped_line = line.lstrip()
+    if not stripped_line or stripped_line.startswith('#'):
+        return line
+
+    # Comment out the names of optional sections.
+    one_indent = ' ' * INDENT
+    if not line.startswith(one_indent):
+        return '#' + line
+
+    # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
+    return '#'.join((one_indent, line[INDENT:]))
+
+
+REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
+REQUIRED_SECTION_NAMES = {'location', 'retention'}
+
+
+def _comment_out_optional_configuration(rendered_config):
     '''
-    Given a target config filename and a config data structure of nested OrderedDicts, write out the
-    config to file as YAML. Create any containing directories as needed.
+    Post-process a rendered configuration string to comment out optional key/values. The idea is
+    that this prevents the user from having to comment out a bunch of configuration they don't care
+    about to get to a minimal viable configuration file.
+
+    Ideally ruamel.yaml would support this during configuration generation, but it's not terribly
+    easy to accomplish that way.
+    '''
+    lines = []
+    required = False
+
+    for line in rendered_config.split('\n'):
+        key = line.strip().split(':')[0]
+
+        if key in REQUIRED_SECTION_NAMES:
+            lines.append(line)
+            continue
+
+        # Upon encountering a required configuration option, skip commenting out lines until the
+        # next blank line.
+        if key in REQUIRED_KEYS:
+            required = True
+        elif not key:
+            required = False
+
+        lines.append(_comment_out_line(line) if not required else line)
+
+    return '\n'.join(lines)
+
+
+def _render_configuration(config):
+    '''
+    Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
+    '''
+    return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)
+
+
+def write_configuration(config_filename, rendered_config, mode=0o600):
+    '''
+    Given a target config filename and rendered config YAML, write it out to file. Create any
+    containing directories as needed.
     '''
     if os.path.exists(config_filename):
         raise FileExistsError('{} already exists. Aborting.'.format(config_filename))
@@ -54,7 +107,7 @@ def write_configuration(config_filename, config, mode=0o600):
         pass
 
     with open(config_filename, 'w') as config_file:
-        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
+        config_file.write(rendered_config)
 
     os.chmod(config_filename, mode)
 
@@ -73,11 +126,7 @@ def add_comments_to_configuration(config, schema, indent=0):
         if not field_schema or not description:
             continue
 
-        config.yaml_set_comment_before_after_key(
-            key=field_name,
-            before=description,
-            indent=indent,
-        )
+        config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
         if index > 0:
             _insert_newline_before_comment(config, field_name)
 
@@ -90,4 +139,6 @@ def generate_sample_configuration(config_filename, schema_filename):
     schema = yaml.round_trip_load(open(schema_filename))
     config = _schema_to_sample_configuration(schema)
 
-    write_configuration(config_filename, config)
+    write_configuration(
+        config_filename, _comment_out_optional_configuration(_render_configuration(config))
+    )

borgmatic/config/legacy.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict, namedtuple
 from configparser import RawConfigParser
 
-
 Section_format = namedtuple('Section_format', ('name', 'options'))
 Config_option = namedtuple('Config_option', ('name', 'value_type', 'required'))
 
@@ -45,12 +44,8 @@ CONFIG_FORMAT = (
         ),
     ),
     Section_format(
-        'consistency',
-        (
-            option('checks', required=False),
-            option('check_last', required=False),
-        ),
-    )
+        'consistency', (option('checks', required=False), option('check_last', required=False))
+    ),
 )
 
 
@@ -66,7 +61,8 @@ def validate_configuration_format(parser, config_format):
     '''
     section_names = set(parser.sections())
     required_section_names = tuple(
-        section.name for section in config_format
+        section.name
+        for section in config_format
         if any(option.required for option in section.options)
     )
 
@@ -80,9 +76,7 @@ def validate_configuration_format(parser, config_format):
 
     missing_section_names = set(required_section_names) - section_names
     if missing_section_names:
-        raise ValueError(
-            'Missing config sections: {}'.format(', '.join(missing_section_names))
-        )
+        raise ValueError('Missing config sections: {}'.format(', '.join(missing_section_names)))
 
     for section_format in config_format:
         if section_format.name not in section_names:
@@ -91,26 +85,28 @@ def validate_configuration_format(parser, config_format):
         option_names = parser.options(section_format.name)
         expected_options = section_format.options
 
-        unexpected_option_names = set(option_names) - set(option.name for option in expected_options)
+        unexpected_option_names = set(option_names) - set(
+            option.name for option in expected_options
+        )
 
         if unexpected_option_names:
             raise ValueError(
                 'Unexpected options found in config section {}: {}'.format(
-                    section_format.name,
-                    ', '.join(sorted(unexpected_option_names)),
+                    section_format.name, ', '.join(sorted(unexpected_option_names))
                 )
             )
 
         missing_option_names = tuple(
-            option.name for option in expected_options if option.required
+            option.name
+            for option in expected_options
+            if option.required
             if option.name not in option_names
         )
 
         if missing_option_names:
             raise ValueError(
                 'Required options missing from config section {}: {}'.format(
-                    section_format.name,
-                    ', '.join(missing_option_names)
+                    section_format.name, ', '.join(missing_option_names)
                 )
             )
 
@@ -123,11 +119,7 @@ def parse_section_options(parser, section_format):
 
     Raise ValueError if any option values cannot be coerced to the expected Python data type.
     '''
-    type_getter = {
-        str: parser.get,
-        int: parser.getint,
-        bool: parser.getboolean,
-    }
+    type_getter = {str: parser.get, int: parser.getint, bool: parser.getboolean}
 
     return OrderedDict(
         (option.name, type_getter[option.value_type](section_format.name, option.name))
@@ -151,11 +143,10 @@ def parse_configuration(config_filename, config_format):
 
     # Describes a parsed configuration, where each attribute is the name of a configuration file
     # section and each value is a dict of that section's parsed options.
-    Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
+    Parsed_config = namedtuple(
+        'Parsed_config', (section_format.name for section_format in config_format)
+    )
 
     return Parsed_config(
-        *(
-            parse_section_options(parser, section_format)
-            for section_format in config_format
-        )
+        *(parse_section_options(parser, section_format) for section_format in config_format)
     )

borgmatic/config/load.py

@@ -0,0 +1,59 @@
+import logging
+import os
+
+import ruamel.yaml
+
+logger = logging.getLogger(__name__)
+
+
+def load_configuration(filename):
+    '''
+    Load the given configuration file and return its contents as a data structure of nested dicts
+    and lists.
+
+    Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
+    if there are too many recursive includes.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.Constructor = Include_constructor
+
+    return yaml.load(open(filename))
+
+
+def include_configuration(loader, filename_node):
+    '''
+    Load the given YAML filename (ignoring the given loader so we can use our own), and return its
+    contents as a data structure of nested dicts and lists.
+    '''
+    return load_configuration(os.path.expanduser(filename_node.value))
+
+
+class Include_constructor(ruamel.yaml.SafeConstructor):
+    '''
+    A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
+    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    '''
+
+    def __init__(self, preserve_quotes=None, loader=None):
+        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        self.add_constructor('!include', include_configuration)
+
+    def flatten_mapping(self, node):
+        '''
+        Support the special case of shallow merging included configuration into an existing mapping
+        using the YAML '<<' merge key. Example syntax:
+
+        ```
+        retention:
+            keep_daily: 1
+            <<: !include common.yaml
+        ```
+        '''
+        representer = ruamel.yaml.representer.SafeRepresenter()
+
+        for index, (key_node, value_node) in enumerate(node.value):
+            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+                included_value = representer.represent_data(self.construct_object(value_node))
+                node.value[index] = (key_node, included_value)
+
+        super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -12,98 +12,226 @@ map:
                 required: true
                 seq:
                     - type: scalar
-                desc: List of source directories to backup (required). Globs are expanded.
+                desc: |
+                    List of source directories to backup (required). Globs and tildes are expanded.
                 example:
                     - /home
                     - /etc
                     - /var/log/syslog*
-            one_file_system:
-                type: bool
-                desc: Stay in same file system (do not cross mount points).
-                example: true
-            remote_path:
-                type: scalar
-                desc: Alternate Borg remote executable. Defaults to "borg".
-                example: borg1
             repositories:
                 required: true
                 seq:
                     - type: scalar
                 desc: |
-                    Paths to local or remote repositories (required). Multiple repositories are
-                    backed up to in sequence.
+                    Paths to local or remote repositories (required). Tildes are expanded. Multiple
+                    repositories are backed up to in sequence. See ssh_command for SSH options like
+                    identity file or port.
                 example:
                     - user@backupserver:sourcehostname.borg
+            one_file_system:
+                type: bool
+                desc: Stay in same file system (do not cross mount points). Defaults to false.
+                example: true
+            numeric_owner:
+                type: bool
+                desc: Only store/extract numeric user and group identifiers. Defaults to false.
+                example: true
+            read_special:
+                type: bool
+                desc: |
+                    Use Borg's --read-special flag to allow backup of block and other special
+                    devices. Use with caution, as it will lead to problems if used when
+                    backing up special devices such as /dev/zero. Defaults to false.
+                example: false
+            bsd_flags:
+                type: bool
+                desc: Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true.
+                example: true
+            files_cache:
+                type: scalar
+                desc: |
+                    Mode in which to operate the files cache. See
+                    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
+                    details. Defaults to "ctime,size,inode".
+                example: ctime,size,inode
+            local_path:
+                type: scalar
+                desc: Alternate Borg local executable. Defaults to "borg".
+                example: borg1
+            remote_path:
+                type: scalar
+                desc: Alternate Borg remote executable. Defaults to "borg".
+                example: borg1
+            patterns:
+                seq:
+                    - type: scalar
+                desc: |
+                    Any paths matching these patterns are included/excluded from backups. Globs are
+                    expanded. (Tildes are not.) Note that Borg considers this option experimental.
+                    See the output of "borg help patterns" for more details. Quote any value if it
+                    contains leading punctuation, so it parses correctly.
+                example:
+                    - 'R /'
+                    - '- /home/*/.cache'
+                    - '+ /home/susan'
+                    - '- /home/*'
+            patterns_from:
+                seq:
+                    - type: scalar
+                desc: |
+                    Read include/exclude patterns from one or more separate named files, one pattern
+                    per line. Note that Borg considers this option experimental. See the output of
+                    "borg help patterns" for more details.
+                example:
+                    - /etc/borgmatic/patterns
             exclude_patterns:
                 seq:
                     - type: scalar
                 desc: |
-                    Any paths matching these patterns are excluded from backups. Globs are expanded.
-                    See https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns
-                    for details.
+                    Any paths matching these patterns are excluded from backups. Globs and tildes
+                    are expanded. See the output of "borg help patterns" for more details.
                 example:
                     - '*.pyc'
-                    - /home/*/.cache
+                    - ~/*/.cache
                     - /etc/ssl
             exclude_from:
                 seq:
                     - type: scalar
                 desc: |
                     Read exclude patterns from one or more separate named files, one pattern per
-                    line.
+                    line. See the output of "borg help patterns" for more details.
                 example:
                     - /etc/borgmatic/excludes
             exclude_caches:
                 type: bool
                 desc: |
                     Exclude directories that contain a CACHEDIR.TAG file. See
-                    http://www.brynosaurus.com/cachedir/spec.html for details.
+                    http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false.
                 example: true
             exclude_if_present:
                 type: scalar
-                desc: Exclude directories that contain a file with the given filename.
+                desc: |
+                    Exclude directories that contain a file with the given filename. Defaults to not
+                    set.
                 example: .nobackup
     storage:
         desc: |
             Repository storage options. See
             https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
-            https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details.
+            https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for
+            details.
         map:
+            encryption_passcommand:
+                type: scalar
+                desc: |
+                    The standard output of this command is used to unlock the encryption key. Only
+                    use on repositories that were initialized with passcommand/repokey encryption.
+                    Note that if both encryption_passcommand and encryption_passphrase are set,
+                    then encryption_passphrase takes precedence. Defaults to not set.
+                example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
                 type: scalar
                 desc: |
                     Passphrase to unlock the encryption key with. Only use on repositories that were
                     initialized with passphrase/repokey encryption. Quote the value if it contains
                     punctuation, so it parses correctly. And backslash any quote or backslash
-                    literals as well.
+                    literals as well. Defaults to not set.
                 example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+            checkpoint_interval:
+                type: int
+                desc: |
+                    Number of seconds between each checkpoint during a long-running backup. See
+                    https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there
+                    for details. Defaults to checkpoints every 1800 seconds (30 minutes).
+                example: 1800
+            chunker_params:
+                type: scalar
+                desc: |
+                    Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
+                    HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
+                    for details. Defaults to "19,23,21,4095".
+                example: 19,23,21,4095
             compression:
                 type: scalar
                 desc: |
                     Type of compression to use when creating archives. See
                     https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
-                    Defaults to no compression.
+                    Defaults to "lz4".
                 example: lz4
+            remote_rate_limit:
+                type: int
+                desc: Remote network upload rate limit in kiBytes/second. Defaults to unlimited.
+                example: 100
+            ssh_command:
+                type: scalar
+                desc: |
+                    Command to use instead of "ssh". This can be used to specify ssh options.
+                    Defaults to not set.
+                example: ssh -i /path/to/private/key
+            borg_base_directory:
+                type: scalar
+                desc: |
+                    Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
+                    See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
+                example: /path/to/base
+            borg_config_directory:
+                type: scalar
+                desc: |
+                    Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
+                example: /path/to/base/config
+            borg_cache_directory:
+                type: scalar
+                desc: |
+                    Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
+                example: /path/to/base/cache
+            borg_security_directory:
+                type: scalar
+                desc: |
+                    Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
+                example: /path/to/base/config/security
+            borg_keys_directory:
+                type: scalar
+                desc: |
+                    Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
+                example: /path/to/base/config/keys
             umask:
                 type: scalar
-                desc: Umask to be used for borg create.
+                desc: Umask to be used for borg create. Defaults to 0077.
                 example: 0077
+            lock_wait:
+                type: int
+                desc: Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1.
+                example: 5
             archive_name_format:
                 type: scalar
                 desc: |
-                    Name of the archive. Borg placeholders can be used. See
-                    https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-placeholders
-                    Default is "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}"
+                    Name of the archive. Borg placeholders can be used. See the output of
+                    "borg help placeholders" for details. Defaults to
+                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
+                    also specify a prefix in the retention section to avoid accidental pruning of
+                    archives with a different archive name format. And you should also specify a
+                    prefix in the consistency section as well.
                 example: "{hostname}-documents-{now}"
     retention:
         desc: |
             Retention policy for how many backups to keep in each category. See
             https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
+            At least one of the "keep" options is required for pruning to work. See
+            https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/
+            if you'd like to skip pruning entirely.
         map:
             keep_within:
                 type: scalar
                 desc: Keep all archives within this time interval.
                 example: 3H
+            keep_secondly:
+                type: int
+                desc: Number of secondly archives to keep.
+                example: 60
+            keep_minutely:
+                type: int
+                desc: Number of minutely archives to keep.
+                example: 60
             keep_hourly:
                 type: int
                 desc: Number of hourly archives to keep.
@@ -128,8 +256,8 @@ map:
                 type: scalar
                 desc: |
                     When pruning, only consider archive names starting with this prefix.
-                    Borg placeholders can be used. See
-                    https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-placeholders
+                    Borg placeholders can be used. See the output of "borg help placeholders" for
+                    details. Defaults to "{hostname}-".
                 example: sourcehostname
     consistency:
         desc: |
@@ -147,12 +275,68 @@ map:
                     "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
                     all consistency checks. "repository" checks the consistency of the repository,
                     "archive" checks all of the archives, and "extract" does an extraction dry-run
-                    of just the most recent archive.
+                    of the most recent archive.
                 example:
                     - repository
                     - archives
+            check_repositories:
+                seq:
+                    - type: scalar
+                desc: |
+                    Paths to a subset of the repositories in the location section on which to run
+                    consistency checks. Handy in case some of your repositories are very large, and
+                    so running consistency checks on them would take too long. Defaults to running
+                    consistency checks on all repositories configured in the location section.
+                example:
+                    - user@backupserver:sourcehostname.borg
             check_last:
                 type: int
                 desc: Restrict the number of checked archives to the last n. Applies only to the
-                      "archives" check.
+                      "archives" check. Defaults to checking all archives.
                 example: 3
+            prefix:
+                type: scalar
+                desc: |
+                    When performing the "archives" check, only consider archive names starting with
+                    this prefix. Borg placeholders can be used. See the output of
+                    "borg help placeholders" for details. Defaults to "{hostname}-".
+                example: sourcehostname
+    output:
+        desc: |
+            Options for customizing borgmatic's own output and logging.
+        map:
+            color:
+                type: bool
+                desc: |
+                    Apply color to console output. Can be overridden with --no-color command-line
+                    flag. Defaults to true.
+                example: false
+    hooks:
+        desc: |
+            Shell commands or scripts to execute before and after a backup or if an error has occurred.
+            IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
+            Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
+            prevent potential shell injection or privilege escalation.
+        map:
+            before_backup:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute before creating a backup.
+                example:
+                    - echo "Starting a backup job."
+            after_backup:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute after creating a backup.
+                example:
+                    - echo "Backup created."
+            on_error:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
+                example:
+                    - echo "Error while creating a backup."
+            umask:
+                type: scalar
+                desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.
+                example: 0077

borgmatic/config/validate.py

@@ -1,11 +1,11 @@
 import logging
-import sys
-import warnings
 
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-from ruamel import yaml
+import ruamel.yaml
+
+from borgmatic.config import load
 
 
 def schema_filename():
@@ -21,10 +21,48 @@ class Validation_error(ValueError):
     A collection of error message strings generated when attempting to validate a particular
     configurartion file.
     '''
+
     def __init__(self, config_filename, error_messages):
         self.config_filename = config_filename
         self.error_messages = error_messages
 
+    def __str__(self):
+        '''
+        Render a validation error as a user-facing string.
+        '''
+        return 'An error occurred while parsing a configuration file at {}:\n'.format(
+            self.config_filename
+        ) + '\n'.join(self.error_messages)
+
+
+def apply_logical_validation(config_filename, parsed_configuration):
+    '''
+    Given a parsed and schematically valid configuration as a data structure of nested dicts (see
+    below), run through any additional logical validation checks. If there are any such validation
+    problems, raise a Validation_error.
+    '''
+    archive_name_format = parsed_configuration.get('storage', {}).get('archive_name_format')
+    prefix = parsed_configuration.get('retention', {}).get('prefix')
+
+    if archive_name_format and not prefix:
+        raise Validation_error(
+            config_filename,
+            ('If you provide an archive_name_format, you must also specify a retention prefix.',),
+        )
+
+    location_repositories = parsed_configuration.get('location', {}).get('repositories')
+    check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
+    for repository in check_repositories:
+        if repository not in location_repositories:
+            raise Validation_error(
+                config_filename,
+                (
+                    'Unknown repository in the consistency section\'s check_repositories: {}'.format(
+                        repository
+                    ),
+                ),
+            )
+
 
 def parse_configuration(config_filename, schema_filename):
     '''
@@ -38,17 +76,19 @@ def parse_configuration(config_filename, schema_filename):
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
+    logging.getLogger('pykwalify').setLevel(logging.ERROR)
+
     try:
-        config = yaml.round_trip_load(open(config_filename))
-        schema = yaml.round_trip_load(open(schema_filename))
-    except yaml.error.YAMLError as error:
+        config = load.load_configuration(config_filename)
+        schema = load.load_configuration(schema_filename)
+    except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
     # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
-    # simply remove all examples before passing the schema to pykwalify.
+    # remove all examples before passing the schema to pykwalify.
     for section_name, section_schema in schema['map'].items():
         for field_name, field_schema in section_schema['map'].items():
-            field_schema.pop('example')
+            field_schema.pop('example', None)
 
     validator = pykwalify.core.Core(source_data=config, schema_data=schema)
     parsed_result = validator.validate(raise_exception=False)
@@ -56,19 +96,49 @@ def parse_configuration(config_filename, schema_filename):
     if validator.validation_errors:
         raise Validation_error(config_filename, validator.validation_errors)
 
+    apply_logical_validation(config_filename, parsed_result)
+
     return parsed_result
 
 
-def display_validation_error(validation_error):
+def guard_configuration_contains_repository(repository, configurations):
     '''
-    Given a Validation_error, display its error messages to stderr.
+    Given a repository path and a dict mapping from config filename to corresponding parsed config
+    dict, ensure that the repository is declared exactly once in all of the configurations.
+
+    If no repository is given, then error if there are multiple configured repositories.
+
+    Raise ValueError if the repository is not found in a configuration, or is declared multiple
+    times.
     '''
-    print(
-        'An error occurred while parsing a configuration file at {}:'.format(
-            validation_error.config_filename
-        ),
-        file=sys.stderr,
+    if not repository:
+        count = len(
+            tuple(
+                config_repository
+                for config in configurations.values()
+                for config_repository in config['location']['repositories']
+            )
+        )
+
+        if count > 1:
+            raise ValueError(
+                'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
+                    repository
+                )
+            )
+
+        return
+
+    count = len(
+        tuple(
+            config_repository
+            for config in configurations.values()
+            for config_repository in config['location']['repositories']
+            if repository == config_repository
+        )
     )
 
-    for error in validation_error.error_messages:
-        print(error, file=sys.stderr)
+    if count == 0:
+        raise ValueError('Repository {} not found in configuration files'.format(repository))
+    if count > 1:
+        raise ValueError('Repository {} found in multiple configuration files'.format(repository))

borgmatic/execute.py

@@ -0,0 +1,43 @@
+import logging
+import subprocess
+
+logger = logging.getLogger(__name__)
+
+
+def execute_and_log_output(full_command, output_log_level, shell):
+    process = subprocess.Popen(
+        full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
+    )
+
+    while process.poll() is None:
+        line = process.stdout.readline().rstrip().decode()
+        if not line:
+            continue
+
+        if line.startswith('borg: error:'):
+            logger.error(line)
+        else:
+            logger.log(output_log_level, line)
+
+    remaining_output = process.stdout.read().rstrip().decode()
+    if remaining_output:  # pragma: no cover
+        logger.log(output_log_level, remaining_output)
+
+    exit_code = process.poll()
+    if exit_code != 0:
+        raise subprocess.CalledProcessError(exit_code, full_command)
+
+
+def execute_command(full_command, output_log_level=logging.INFO, shell=False):
+    '''
+    Execute the given command (a sequence of command/argument strings) and log its output at the
+    given log level. If output log level is None, instead capture and return the output. If
+    shell is True, execute the command within a shell.
+    '''
+    logger.debug(' '.join(full_command))
+
+    if output_log_level is None:
+        output = subprocess.check_output(full_command, shell=shell)
+        return output.decode() if output is not None else None
+    else:
+        execute_and_log_output(full_command, output_log_level, shell=shell)

borgmatic/hook.py

@@ -0,0 +1,53 @@
+import logging
+import os
+
+from borgmatic import execute
+
+logger = logging.getLogger(__name__)
+
+
+def execute_hook(commands, umask, config_filename, description, dry_run):
+    '''
+    Given a list of hook commands to execute, a umask to execute with (or None), a config filename,
+    a hook description, and whether this is a dry run, run the given commands. Or, don't run them
+    if this is a dry run.
+
+    Raise ValueError if the umask cannot be parsed.
+    '''
+    if not commands:
+        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
+        return
+
+    dry_run_label = ' (dry run; not actually running hooks)' if dry_run else ''
+
+    if len(commands) == 1:
+        logger.info(
+            '{}: Running command for {} hook{}'.format(config_filename, description, dry_run_label)
+        )
+    else:
+        logger.info(
+            '{}: Running {} commands for {} hook{}'.format(
+                config_filename, len(commands), description, dry_run_label
+            )
+        )
+
+    if umask:
+        parsed_umask = int(str(umask), 8)
+        logger.debug('{}: Set hook umask to {}'.format(config_filename, oct(parsed_umask)))
+        original_umask = os.umask(parsed_umask)
+    else:
+        original_umask = None
+
+    try:
+        for command in commands:
+            if not dry_run:
+                execute.execute_command(
+                    [command],
+                    output_log_level=logging.ERROR
+                    if description == 'on-error'
+                    else logging.WARNING,
+                    shell=True,
+                )
+    finally:
+        if original_umask:
+            os.umask(original_umask)

borgmatic/logger.py

@@ -0,0 +1,93 @@
+import logging
+import os
+import sys
+
+import colorama
+
+
+def to_bool(arg):
+    '''
+    Return a boolean value based on `arg`.
+    '''
+    if arg is None or isinstance(arg, bool):
+        return arg
+
+    if isinstance(arg, str):
+        arg = arg.lower()
+
+    if arg in ('yes', 'on', '1', 'true', 1):
+        return True
+
+    return False
+
+
+def should_do_markup(no_color, configs):
+    '''
+    Given the value of the command-line no-color argument, and a dict of configuration filename to
+    corresponding parsed configuration, determine if we should enable colorama marking up.
+    '''
+    if no_color:
+        return False
+
+    if any(config.get('output', {}).get('color') is False for config in configs.values()):
+        return False
+
+    py_colors = os.environ.get('PY_COLORS', None)
+
+    if py_colors is not None:
+        return to_bool(py_colors)
+
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+
+
+LOG_LEVEL_TO_COLOR = {
+    logging.CRITICAL: colorama.Fore.RED,
+    logging.ERROR: colorama.Fore.RED,
+    logging.WARN: colorama.Fore.YELLOW,
+    logging.INFO: colorama.Fore.GREEN,
+    logging.DEBUG: colorama.Fore.CYAN,
+}
+
+
+class Console_color_formatter(logging.Formatter):
+    def format(self, record):
+        color = LOG_LEVEL_TO_COLOR.get(record.levelno)
+        return color_text(color, record.msg)
+
+
+def color_text(color, message):
+    '''
+    Give colored text.
+    '''
+    if not color:
+        return message
+
+    return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
+
+
+def configure_logging(console_log_level, syslog_log_level=None):
+    '''
+    Configure logging to go to both the console and syslog. Use the given log levels, respectively.
+    '''
+    if syslog_log_level is None:
+        syslog_log_level = console_log_level
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(Console_color_formatter())
+    console_handler.setLevel(console_log_level)
+
+    syslog_path = None
+    if os.path.exists('/dev/log'):
+        syslog_path = '/dev/log'
+    elif os.path.exists('/var/run/syslog'):
+        syslog_path = '/var/run/syslog'
+
+    if syslog_path:
+        syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
+        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s \ufeff%(message)s'))
+        syslog_handler.setLevel(syslog_log_level)
+        handlers = (console_handler, syslog_handler)
+    else:
+        handlers = (console_handler,)
+
+    logging.basicConfig(level=min(console_log_level, syslog_log_level), handlers=handlers)

borgmatic/signals.py

@@ -0,0 +1,18 @@
+import os
+import signal
+
+
+def _handle_signal(signal_number, frame):  # pragma: no cover
+    '''
+    Send the signal to all processes in borgmatic's process group, which includes child process.
+    '''
+    os.killpg(os.getpgrp(), signal_number)
+
+
+def configure_signals():  # pragma: no cover
+    '''
+    Configure borgmatic's signal handlers to pass relevant signals through to any child processes
+    like Borg. Note that SIGINT gets passed through even without these changes.
+    '''
+    for signal_number in (signal.SIGHUP, signal.SIGTERM, signal.SIGUSR1, signal.SIGUSR2):
+        signal.signal(signal_number, _handle_signal)

borgmatic/tests/integration/commands/test_borgmatic.py

@@ -1,66 +0,0 @@
-import os
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic.commands import borgmatic as module
-
-
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    parser = module.parse_arguments()
-
-    assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
-    assert parser.excludes_filename == None
-    assert parser.verbosity is None
-
-
-def test_parse_arguments_with_path_arguments_overrides_defaults():
-    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
-
-    assert parser.config_paths == ['myconfig']
-    assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity is None
-
-
-def test_parse_arguments_with_multiple_config_paths_parses_as_list():
-    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
-
-    assert parser.config_paths == ['myconfig', 'otherconfig']
-    assert parser.verbosity is None
-
-
-def test_parse_arguments_with_verbosity_flag_overrides_default():
-    parser = module.parse_arguments('--verbosity', '1')
-
-    assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
-    assert parser.excludes_filename == None
-    assert parser.verbosity == 1
-
-
-def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
-    parser = module.parse_arguments()
-
-    assert parser.prune is True
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
-    parser = module.parse_arguments('--prune')
-
-    assert parser.prune is True
-    assert parser.create is False
-    assert parser.check is False
-
-
-def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
-    parser = module.parse_arguments('--create', '--check')
-
-    assert parser.prune is False
-    assert parser.create is True
-    assert parser.check is True
-
-
-def test_parse_arguments_with_invalid_arguments_exits():
-    with pytest.raises(SystemExit):
-        module.parse_arguments('--posix-me-harder')

borgmatic/tests/integration/config/test_validate.py

@@ -1,123 +0,0 @@
-import io
-import string
-import sys
-import os
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic.config import validate as module
-
-
-def test_schema_filename_returns_plausable_path():
-    schema_path = module.schema_filename()    
-
-    assert schema_path.endswith('/schema.yaml')
-
-
-def mock_config_and_schema(config_yaml):
-    '''
-    Set up mocks for the config config YAML string and the default schema so that the code under
-    test consumes them when parsing the configuration.
-    '''
-    config_stream = io.StringIO(config_yaml)
-    schema_stream = open(module.schema_filename())
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').with_args('config.yaml').and_return(config_stream)
-    builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream)
-
-
-def test_parse_configuration_transforms_file_into_mapping():
-    mock_config_and_schema(
-        '''
-        location:
-            source_directories:
-                - /home
-                - /etc
-
-            repositories:
-                - hostname.borg
-
-        retention:
-            keep_daily: 7
-
-        consistency:
-            checks:
-                - repository
-                - archives
-        '''
-    )
-
-    result = module.parse_configuration('config.yaml', 'schema.yaml')
-
-    assert result == {
-        'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
-        'retention': {'keep_daily': 7},
-        'consistency': {'checks': ['repository', 'archives']},
-    }
-
-
-def test_parse_configuration_passes_through_quoted_punctuation():
-    escaped_punctuation = string.punctuation.replace('\\', r'\\').replace('"', r'\"')
-
-    mock_config_and_schema(
-        '''
-        location:
-            source_directories:
-                - /home
-
-            repositories:
-                - "{}.borg"
-        '''.format(escaped_punctuation)
-    )
-
-    result = module.parse_configuration('config.yaml', 'schema.yaml')
-
-    assert result == {
-        'location': {
-            'source_directories': ['/home'],
-            'repositories': ['{}.borg'.format(string.punctuation)],
-        },
-    }
-
-
-def test_parse_configuration_raises_for_missing_config_file():
-    with pytest.raises(FileNotFoundError):
-        module.parse_configuration('config.yaml', 'schema.yaml')
-
-
-def test_parse_configuration_raises_for_missing_schema_file():
-    mock_config_and_schema('')
-    builtins = flexmock(sys.modules['builtins'])
-    builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)
-
-    with pytest.raises(FileNotFoundError):
-        module.parse_configuration('config.yaml', 'schema.yaml')
-
-
-def test_parse_configuration_raises_for_syntax_error():
-    mock_config_and_schema('foo:\nbar')
-
-    with pytest.raises(ValueError):
-        module.parse_configuration('config.yaml', 'schema.yaml')
-
-
-def test_parse_configuration_raises_for_validation_error():
-    mock_config_and_schema(
-        '''
-        location:
-            source_directories: yes
-            repositories:
-                - hostname.borg
-        '''
-    )
-
-    with pytest.raises(module.Validation_error):
-        module.parse_configuration('config.yaml', 'schema.yaml')
-
-
-def test_display_validation_error_does_not_raise():
-    flexmock(sys.modules['builtins']).should_receive('print')
-    error = module.Validation_error('config.yaml', ('oops', 'uh oh'))
-
-    module.display_validation_error(error)

borgmatic/tests/integration/test_version.py

@@ -1,8 +0,0 @@
-import subprocess
-
-
-def test_setup_version_matches_news_version():
-    setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii')
-    news_version = open('NEWS').readline()
-
-    assert setup_version == news_version

borgmatic/tests/unit/borg/test_check.py

@@ -1,185 +0,0 @@
-from subprocess import STDOUT
-import sys
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic.borg import check as module
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
-def insert_subprocess_never():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
-
-
-def test_parse_checks_returns_them_as_tuple():
-    checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
-
-    assert checks == ('foo', 'bar')
-
-
-def test_parse_checks_with_missing_value_returns_defaults():
-    checks = module._parse_checks({})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_blank_value_returns_defaults():
-    checks = module._parse_checks({'checks': []})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_disabled_returns_no_checks():
-    checks = module._parse_checks({'checks': ['disabled']})
-
-    assert checks == ()
-
-
-def test_make_check_flags_with_checks_returns_flags():
-    flags = module._make_check_flags(('repository',))
-
-    assert flags == ('--repository-only',)
-
-
-def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
-    flags = module._make_check_flags(('extract',))
-
-    assert flags == ()
-
-
-def test_make_check_flags_with_default_checks_returns_no_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS)
-
-    assert flags == ()
-
-
-def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
-    flags = module._make_check_flags(('repository',), check_last=3)
-
-    assert flags == ('--repository-only', '--last', '3')
-
-
-def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
-
-    assert flags == ('--last', '3')
-
-
-@pytest.mark.parametrize(
-    'checks',
-    (
-        ('repository',),
-        ('archives',),
-        ('repository', 'archives'),
-        ('repository', 'archives', 'other'),
-    ),
-)
-def test_check_archives_should_call_borg_with_parameters(checks):
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo'),
-        stdout=stdout, stderr=STDOUT,
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_extract_check_should_call_extract_only():
-    checks = ('extract',)
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').never()
-    flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
-    insert_subprocess_never()
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
-    checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--info'),
-        stdout=None, stderr=STDOUT,
-    )
-
-    module.check_archives(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--debug'),
-        stdout=None, stderr=STDOUT,
-    )
-
-    module.check_archives(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_without_any_checks_should_bail():
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(())
-    insert_subprocess_never()
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-    )
-
-
-def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
-    checks = ('repository',)
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('borg', 'check', 'repo', '--remote-path', 'borg1'),
-        stdout=stdout, stderr=STDOUT,
-    )
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        remote_path='borg1',
-    )

borgmatic/tests/unit/borg/test_create.py

@@ -1,353 +0,0 @@
-import os
-
-from flexmock import flexmock
-
-from borgmatic.borg import create as module
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def test_initialize_with_passphrase_should_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({'encryption_passphrase': 'pass'})
-        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_without_passphrase_should_not_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({})
-        assert os.environ.get('BORG_PASSPHRASE') == None
-    finally:
-        os.environ = orig_environ
-
-
-def test_write_exclude_file_does_not_raise():
-    temporary_file = flexmock(
-        name='filename',
-        write=lambda mode: None,
-        flush=lambda: None,
-    )
-    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
-
-    module._write_exclude_file(['exclude'])
-
-
-def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
-    module._write_exclude_file([])
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
-def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_patterns': ['*.pyc', '/var']},
-        exclude_patterns_filename='/tmp/excludes',
-    )
-
-    assert exclude_flags == ('--exclude-from', '/tmp/excludes')
-
-
-def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_from': ['excludes', 'other']},
-    )
-
-    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other')
-
-
-def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_from': ['excludes']},
-        exclude_patterns_filename='/tmp/excludes',
-    )
-
-    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
-
-
-def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_from': None},
-    )
-
-    assert exclude_flags == ()
-
-
-def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_caches': True},
-    )
-
-    assert exclude_flags == ('--exclude-caches',)
-
-
-def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_caches': False},
-    )
-
-    assert exclude_flags == ()
-
-
-def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
-    exclude_flags = module._make_exclude_flags(
-        location_config={'exclude_if_present': 'exclude_me'},
-    )
-
-    assert exclude_flags == ('--exclude-if-present', 'exclude_me')
-
-
-def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
-    exclude_flags = module._make_exclude_flags(location_config={})
-
-    assert exclude_flags == ()
-
-
-DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
-
-
-def test_create_archive_should_call_borg_with_parameters():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
-    exclude_flags = ('--exclude-from', 'excludes')
-    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
-    flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
-    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': ['exclude'],
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
-
-    module.create_archive(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
-
-    module.create_archive(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={'compression': 'rle'},
-    )
-
-
-def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'one_file_system': True,
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'remote_path': 'borg1',
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={'umask': 740},
-    )
-
-
-def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
-    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo*'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
-    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo*'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
-    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo*'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={},
-    )
-
-
-def test_create_archive_with_archive_name_format_without_placeholders():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={
-            'archive_name_format': 'ARCHIVE_NAME',
-        },
-    )
-
-
-def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-    flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
-
-    module.create_archive(
-        verbosity=None,
-        repository='repo',
-        location_config={
-            'source_directories': ['foo', 'bar'],
-            'repositories': ['repo'],
-            'exclude_patterns': None,
-        },
-        storage_config={
-            'archive_name_format': 'Documents_{hostname}-{now}',
-        },
-    )

borgmatic/tests/unit/borg/test_extract.py

@@ -1,100 +0,0 @@
-import sys
-
-from flexmock import flexmock
-
-from borgmatic.borg import extract as module
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
-def insert_subprocess_never():
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').never()
-
-
-def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
-
-
-def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=None,
-        repository='repo',
-    )
-
-
-def test_extract_last_archive_dry_run_without_any_archives_should_bail():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo'),
-        result='\n'.encode('utf-8'),
-    )
-    insert_subprocess_never()
-
-    module.extract_last_archive_dry_run(
-        verbosity=None,
-        repository='repo',
-    )
-
-
-def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--info'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-    )
-
-
-def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--debug'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-    )
-
-
-def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
-    flexmock(sys.stdout).encoding = 'utf-8'
-    insert_subprocess_check_output_mock(
-        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
-        result='archive1\narchive2\n'.encode('utf-8'),
-    )
-    insert_subprocess_mock(
-        ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
-    )
-
-    module.extract_last_archive_dry_run(
-        verbosity=None,
-        repository='repo',
-        remote_path='borg1',
-    )

borgmatic/tests/unit/borg/test_prune.py

@@ -1,112 +0,0 @@
-from collections import OrderedDict
-
-from flexmock import flexmock
-
-from borgmatic.borg import prune as module
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
-BASE_PRUNE_FLAGS = (
-    ('--keep-daily', '1'),
-    ('--keep-weekly', '2'),
-    ('--keep-monthly', '3'),
-)
-
-
-def test_make_prune_flags_should_return_flags_from_config():
-    retention_config = OrderedDict(
-        (
-            ('keep_daily', 1),
-            ('keep_weekly', 2),
-            ('keep_monthly', 3),
-        )
-    )
-
-    result = module._make_prune_flags(retention_config)
-
-    assert tuple(result) == BASE_PRUNE_FLAGS
-
-
-def test_make_prune_flags_accepts_prefix_with_placeholders():
-    retention_config = OrderedDict(
-        (
-            ('keep_daily', 1),
-            ('prefix', 'Documents_{hostname}-{now}'),
-        )
-    )
-
-    result = module._make_prune_flags(retention_config)
-
-    expected = (
-        ('--keep-daily', '1'),
-        ('--prefix', 'Documents_{hostname}-{now}'),
-    )
-
-    assert tuple(result) == expected
-
-
-PRUNE_COMMAND = (
-    'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
-)
-
-
-def test_prune_archives_should_call_borg_with_parameters():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND)
-
-    module.prune_archives(
-        verbosity=None,
-        repository='repo',
-        retention_config=retention_config,
-    )
-
-
-def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',))
-
-    module.prune_archives(
-        repository='repo',
-        verbosity=VERBOSITY_SOME,
-        retention_config=retention_config,
-    )
-
-
-def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list'))
-
-    module.prune_archives(
-        repository='repo',
-        verbosity=VERBOSITY_LOTS,
-        retention_config=retention_config,
-    )
-
-
-def test_prune_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
-
-    module.prune_archives(
-        verbosity=None,
-        repository='repo',
-        retention_config=retention_config,
-        remote_path='borg1',
-    )

borgmatic/tests/unit/config/test_collect.py

@@ -1,58 +0,0 @@
-from flexmock import flexmock
-
-from borgmatic.config import collect as module
-
-
-def test_collect_config_filenames_collects_given_files():
-    config_paths = ('config.yaml', 'other.yaml')
-    flexmock(module.os.path).should_receive('isdir').and_return(False)
-
-    config_filenames = tuple(module.collect_config_filenames(config_paths))
-
-    assert config_filenames == config_paths
-
-
-def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
-    config_paths = ('config.yaml', '/etc/borgmatic.d')
-    mock_path = flexmock(module.os.path)
-    mock_path.should_receive('exists').and_return(True)
-    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
-    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
-    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
-    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
-    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
-    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml'])
-
-    config_filenames = tuple(module.collect_config_filenames(config_paths))
-
-    assert config_filenames == (
-        'config.yaml',
-        '/etc/borgmatic.d/foo.yaml',
-        '/etc/borgmatic.d/baz.yaml',
-    )
-
-
-def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
-    config_paths = ('config.yaml', '/etc/borgmatic.d')
-    mock_path = flexmock(module.os.path)
-    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
-    mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False)
-    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
-    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
-
-    config_filenames = tuple(module.collect_config_filenames(config_paths))
-
-    assert config_filenames == ('config.yaml',)
-
-
-def test_collect_config_filenames_includes_directory_if_it_does_not_exist():
-    config_paths = ('config.yaml', '/my/directory')
-    mock_path = flexmock(module.os.path)
-    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
-    mock_path.should_receive('exists').with_args('/my/directory').and_return(False)
-    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
-    mock_path.should_receive('isdir').with_args('/my/directory').and_return(True)
-
-    config_filenames = tuple(module.collect_config_filenames(config_paths))
-
-    assert config_filenames == config_paths

borgmatic/tests/unit/config/test_generate.py

@@ -1,49 +0,0 @@
-from collections import OrderedDict
-
-from flexmock import flexmock
-
-from borgmatic.config import generate as module
-
-
-def test_schema_to_sample_configuration_generates_config_with_examples():
-    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module).should_receive('add_comments_to_configuration')
-    schema = {
-        'map': OrderedDict([
-            (
-                'section1', {
-                    'map': {
-                        'field1': OrderedDict([
-                            ('example', 'Example 1')
-                        ]),
-                    },
-                },
-            ),
-            (
-                'section2', {
-                    'map': OrderedDict([
-                        ('field2', {'example': 'Example 2'}),
-                        ('field3', {'example': 'Example 3'}),
-                    ]),
-                }
-            ),
-        ])
-    }
-
-    config = module._schema_to_sample_configuration(schema)
-
-    assert config == OrderedDict([
-        (
-            'section1',
-            OrderedDict([
-                ('field1', 'Example 1'),
-            ]),
-        ),
-        (
-            'section2',
-            OrderedDict([
-                ('field2', 'Example 2'),
-                ('field3', 'Example 3'),
-            ]),
-        )
-    ])

borgmatic/verbosity.py

@@ -1,2 +1,16 @@
+import logging
+
+VERBOSITY_WARNING = 0
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2
+
+
+def verbosity_to_log_level(verbosity):
+    '''
+    Given a borgmatic verbosity value, return the corresponding Python log level.
+    '''
+    return {
+        VERBOSITY_WARNING: logging.WARNING,
+        VERBOSITY_SOME: logging.INFO,
+        VERBOSITY_LOTS: logging.DEBUG,
+    }.get(verbosity, logging.WARNING)

docs/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.7.3-alpine3.9 as borgmatic
+
+COPY . /app
+RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
+RUN borgmatic --help > /command-line.txt
+
+FROM node:11.15.0-alpine as html
+
+WORKDIR /source
+
+RUN npm install @11ty/eleventy \
+    @11ty/eleventy-plugin-syntaxhighlight \
+    @11ty/eleventy-plugin-inclusive-language \
+    markdown-it \
+    markdown-it-anchor \
+    markdown-it-replace-link
+COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml
+COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt
+COPY . /source
+RUN npx eleventy --input=/source/docs --output=/output/docs \
+  && mv /output/docs/index.html /output/index.html
+
+FROM nginx:1.16.0-alpine
+
+COPY --from=html /output /usr/share/nginx/html
+COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/README.md

@@ -0,0 +1 @@
+../README.md

docs/_data/layout.json

@@ -0,0 +1 @@
+"layouts/main.njk"

docs/_includes/asciinema.css

@@ -0,0 +1,3 @@
+.asciicast > iframe {
+    width: 100% !important;
+}

docs/_includes/components/external-links.css

@@ -0,0 +1,12 @@
+/* External links */
+a[href^="http://"]:not(.minilink):not(.elv-externalexempt),
+a[href^="https://"]:not(.minilink):not(.elv-externalexempt), 
+a[href^="//"]:not(.minilink):not(.elv-externalexempt) {
+	text-decoration-color: inherit;
+}
+/* External link hovers */
+a[href^="http://"]:not(.minilink):not(.elv-externalexempt):hover,
+a[href^="https://"]:not(.minilink):not(.elv-externalexempt):hover, 
+a[href^="//"]:not(.minilink):not(.elv-externalexempt):hover {
+	text-decoration-color: #00bcd4;
+}

docs/_includes/components/info-blocks.css

@@ -0,0 +1,34 @@
+/* Warning */
+.elv-info {
+	line-height: 1.5;
+	padding: 0.8125em 1em 0.75em; /* 13px 16px 12px /16 */
+	margin-left: -1rem;
+	margin-right: -1rem;
+	margin-bottom: 2em;
+	background-color: #dff7ff;
+}
+.elv-info:before {
+	content: "ℹ️ ";
+}
+.elv-info-warn {
+	background-color: #ffa;
+}
+.elv-info-warn:before {
+	content: "⚠️ ";
+}
+.elv-info:first-child {
+	margin-top: 0;
+}
+body > .elv-info {
+	margin-left: 0;
+	margin-right: 0;
+	padding: .5rem 1rem;
+}
+@media (min-width: 37.5em) and (min-height: 25em) { /* 600px / 400px */
+	body > .elv-info-sticky {
+		position: sticky;
+		top: 0;
+		z-index: 2;
+		box-shadow: 0 3px 0 0 rgba(0,0,0,.08);
+	}
+}

docs/_includes/components/lists.css

@@ -0,0 +1,126 @@
+/* Buzzwords */
+@keyframes rainbow {
+	0% { background-position: 0% 50%; }
+	50% { background-position: 100% 50%; }
+	100% { background-position: 0% 50%; }
+}
+.buzzword-list,
+.inlinelist {
+	padding: 0;
+}
+.inlinelist:first-child:last-child {
+	margin: 0;
+}
+.buzzword,
+.buzzword-list li,
+.inlinelist .inlinelist-item {
+	display: inline;
+	-webkit-box-decoration-break: clone;
+	box-decoration-break: clone;
+	font-family: Georgia, serif;
+	font-size: 116%;
+	white-space: normal;
+	line-height: 1.85;
+	padding: .2em .5em;
+	margin: 4px 4px 4px 0;
+	transition: .15s linear outline;
+}
+.inlinelist .inlinelist-item.active {
+	background-color: #222;
+	color: #fff;
+	font-weight: inherit;
+}
+.inlinelist .inlinelist-item.active :link,
+.inlinelist .inlinelist-item.active :visited {
+	color: #fff;
+}
+.inlinelist .inlinelist-item code {
+	background-color: transparent;
+}
+a.buzzword {
+	text-decoration: underline;
+}
+.buzzword-list a,
+.inlinelist a {
+	text-decoration: none;
+}
+.inlinelist .inlinelist-item {
+	font-size: 100%;
+	line-height: 2;
+}
+@supports not(-webkit-box-decoration-break: clone) {
+	.buzzword,
+	.buzzword-list li,
+	.inlinelist .inlinelist-item {
+		display: inline-block;
+	}
+}
+.buzzword-list li,
+.buzzword {
+	background-color: #f7f7f7;
+}
+.inlinelist .inlinelist-item {
+	background-color: #e9e9e9;
+}
+.inlinelist .inlinelist-item:hover,
+.inlinelist .inlinelist-item:focus,
+.buzzword-list li:hover,
+.buzzword-list li:focus,
+.buzzword:hover,
+.buzzword:focus {
+	position: relative;
+	background-image: linear-gradient(238deg, #ff0000, #ff8000, #ffff00, #80ff00, #00ff00, #00ff80, #00ffff, #0080ff, #0000ff, #8000ff, #ff0080);
+	background-size: 1200% 1200%;
+	color: #fff;
+	text-shadow: 0 0 2px rgba(0,0,0,.9);
+	animation: rainbow 1.6s infinite;
+}
+.inlinelist .inlinelist-item:hover a,
+.inlinelist .inlinelist-item:focus a,
+.buzzword-list li:hover a,
+.buzzword-list li:focus a,
+a.buzzword:hover,
+a.buzzword:focus {
+	color: #fff;
+	text-decoration: none;
+}
+/*
+I wish there were a PE friendly way to do this but media queries don’t work work with @supports
+@media (prefers-reduced-motion: no-preference) {
+	.buzzword:hover,
+	.buzzword:focus {
+		animation: rainbow 1s infinite;
+	}
+}*/
+.buzzword-list li:hover:after,
+.buzzword-list li:focus:after,
+.buzzword:hover:after,
+.buzzword:focus:after {
+	font-family: system-ui, sans-serif;
+	content: "Buzzword alert!!!";
+	position: absolute;
+	left: 0;
+	top: 0;
+	max-width: 8em;
+	color: #f00;
+	font-weight: 700;
+	text-transform: uppercase;
+	transform: rotate(-10deg) translate(-25%, -125%);
+	text-shadow: 1px 1px 5px rgba(0,0,0,.6);
+	line-height: 1.2;
+	pointer-events: none;
+}
+main h2 .buzzword,
+main h3 .buzzword,
+main p .buzzword {
+	padding: 0px 7px;
+	font-size: 1em; /* 18px /18 */
+	margin: 0;
+	line-height: 1.444444444444; /* 26px /18 */
+	font-family: inherit;
+}
+main h2 a.buzzword,
+main h3 a.buzzword,
+main p a.buzzword {
+	text-decoration: underline;
+}

docs/_includes/components/minilink.css

@@ -0,0 +1,40 @@
+/* Mini link */
+.minilink {
+	display: inline-block;
+	padding: .125em .375em;
+	text-transform: uppercase;
+	font-size: 0.875rem; /* 14px /16 */
+	text-decoration: none;
+	background-color: #ddd;
+	border-radius: 0.1875em; /* 3px /16 */
+	font-weight: 500;
+	margin: 0 0.4285714285714em 0.07142857142857em 0; /* 0 6px 1px 0 /14 */
+	line-height: 1.285714285714; /* 18px /14 */
+	font-family: system-ui, sans-serif;
+}
+.minilink[href] {
+	box-shadow: 0 1px 1px 0 rgba(0,0,0,.5);
+}
+.minilink[href]:hover,
+.minilink[href]:focus {
+	background-color: #bbb;
+}
+pre + .minilink {
+	color: #fff;
+	border-radius: 0 0 0.2857142857143em 0.2857142857143em; /* 4px /14 */
+	float: right;
+	background-color: #444;
+	color: #fff;
+}
+pre[class*=language-] + .minilink {
+	position: relative;
+	top: -0.7142857142857em; /* -10px /14 */
+}
+p.minilink {
+	float: right;
+	margin-left: 2em;
+	margin-bottom: 2em;
+}
+.minilink + pre[class*=language-] {
+	clear: both;
+}

docs/_includes/components/toc.css

@@ -0,0 +1,63 @@
+.elv-toc {
+	font-size: 1rem; /* Reset */
+}
+@media (min-width: 64em) { /* 1024px */
+	.elv-toc {
+		position: absolute;
+		left: -17rem;
+		width: 16rem;
+	}
+}
+
+
+.elv-toc-list {
+	padding-left: 0;
+	padding-right: 0;
+	list-style: none;
+}
+/* Nested lists */
+.elv-toc-list ul {
+	padding: 0;
+	display: none;
+	margin-bottom: 1.5em;
+	list-style: none;
+}
+.elv-toc-list ul li {
+	padding-left: 0.875em; /* 14px /16 */
+}
+@media (min-width: 64em) and (min-height: 48em) { /* 1024 x 768px */
+	.elv-toc-list ul {
+		display: block;
+	}
+}
+
+/* List items */
+.elv-toc-list a:not(:hover) {
+	text-decoration: none;
+}
+.elv-toc-list li {
+	padding-top: 0;
+	padding-bottom: 0;
+	margin: .1em 0 .5em;
+}
+/* Top level links */
+.elv-toc-list > li > a {
+	font-weight: 400;
+	font-size: 1.0625em; /* 17px /16 */
+	color: #222;
+}
+
+/* Active links */
+.elv-toc-list li.elv-toc-active > a {
+	font-weight: 700;
+	text-decoration: underline;
+}
+.elv-toc-active > a:after {
+	content: " ⬅";
+	line-height: .5;
+}
+/* Show only active nested lists */
+.elv-toc-list ul.elv-toc-active,
+.elv-toc-list li.elv-toc-active > ul {
+	display: block;
+}

docs/_includes/header.njk

@@ -0,0 +1,3 @@
+<header class="elv-layout elv-layout-full elv-header{% if headerClass %} {{ headerClass }}{% endif %}">
+	<h1 class="elv-hed">{{ title | safe }}</h1>
+</header>

docs/_includes/layouts/base.njk

@@ -0,0 +1,27 @@
+<!doctype html>
+<html lang="en"{% if templateClass %} class="{{ templateClass }}"{% endif %}>
+	<head>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0">
+		<title>{{ subtitle + ' - ' if subtitle}}{{ title }}</title>
+{%- set css %}
+{% include 'index.css' %}
+{% include 'components/lists.css' %}
+{% include 'components/external-links.css' %}
+{% include 'components/minilink.css' %}
+{% include 'components/toc.css' %}
+{% include 'components/info-blocks.css' %}
+{% include 'prism-theme.css' %}
+{% include 'asciinema.css' %}
+{% endset %}
+		<style>{{ css | safe }}</style>
+{% if feedTitle and feedUrl %}
+		<link rel="alternate" href="{{ feedUrl }}" title="{{ feedTitle }}" type="application/atom+xml">
+{% endif %}
+	</head>
+	<body>
+
+		{{ content | safe }}
+
+	</body>
+</html>

docs/_includes/layouts/main.njk

@@ -0,0 +1,12 @@
+---
+layout: layouts/base.njk
+templateClass: elv-default
+headerClass: elv-header-default
+---
+{% include "header.njk" %}
+
+<main class="elv-layout{% if layoutClass %} {{ layoutClass }}{% endif %}">
+    <article>
+        {{ content | safe }}
+    </article>
+</main>

docs/_includes/prism-theme.css

@@ -0,0 +1,172 @@
+/**
+ * prism.js default theme for JavaScript, CSS and HTML
+ * Based on dabblet (http://dabblet.com)
+ * @author Lea Verou
+ */
+code[class*="language-"],
+pre[class*="language-"] {
+  color: #ABB2BF;
+  background: none;
+  font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
+  text-align: left;
+  white-space: pre;
+  word-spacing: normal;
+  word-break: normal;
+  word-wrap: normal;
+  line-height: 1.5;
+  -moz-tab-size: 4;
+  -o-tab-size: 4;
+  tab-size: 4;
+  -webkit-hyphens: none;
+  -moz-hyphens: none;
+  -ms-hyphens: none;
+  hyphens: none;
+}
+
+pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection,
+code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection {
+  text-shadow: none;
+  background: #383e49;
+}
+
+pre[class*="language-"]::selection, pre[class*="language-"] ::selection,
+code[class*="language-"]::selection, code[class*="language-"] ::selection {
+  text-shadow: none;
+  background: #9aa2b1;
+}
+
+@media print {
+  code[class*="language-"],
+  pre[class*="language-"] {
+    text-shadow: none;
+  }
+}
+/* Code blocks */
+pre[class*="language-"] {
+  padding: 1em;
+  margin: .5em 0;
+  overflow: auto;
+}
+
+:not(pre) > code[class*="language-"],
+pre[class*="language-"] {
+  background: #282c34;
+}
+
+/* Inline code */
+:not(pre) > code[class*="language-"] {
+  padding: .1em;
+  border-radius: .3em;
+  white-space: normal;
+}
+
+.token.comment,
+.token.prolog,
+.token.doctype,
+.token.cdata {
+  color: #5C6370;
+}
+
+.token.punctuation {
+  color: #abb2bf;
+}
+
+.token.selector,
+.token.tag {
+  color: #e06c75;
+}
+
+.token.property,
+.token.boolean,
+.token.number,
+.token.constant,
+.token.symbol,
+.token.attr-name,
+.token.deleted {
+  color: #d19a66;
+}
+
+.token.string,
+.token.char,
+.token.attr-value,
+.token.builtin,
+.token.inserted {
+  color: #98c379;
+}
+
+.token.operator,
+.token.entity,
+.token.url,
+.language-css .token.string,
+.style .token.string {
+  color: #56b6c2;
+}
+
+.token.atrule,
+.token.keyword {
+  color: #e06c75;
+}
+
+.token.function {
+  color: #61afef;
+}
+
+.token.regex,
+.token.important,
+.token.variable {
+  color: #c678dd;
+}
+
+.token.important,
+.token.bold {
+  font-weight: bold;
+}
+
+.token.italic {
+  font-style: italic;
+}
+
+.token.entity {
+  cursor: help;
+}
+
+pre.line-numbers {
+	position: relative;
+	padding-left: 3.8em;
+	counter-reset: linenumber;
+}
+
+pre.line-numbers > code {
+	position: relative;
+}
+
+.line-numbers .line-numbers-rows {
+	position: absolute;
+	pointer-events: none;
+	top: 0;
+	font-size: 100%;
+	left: -3.8em;
+	width: 3em; /* works for line-numbers below 1000 lines */
+	letter-spacing: -1px;
+	border-right: 0;
+
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	user-select: none;
+
+}
+
+.line-numbers-rows > span {
+	pointer-events: none;
+	display: block;
+	counter-increment: linenumber;
+}
+
+.line-numbers-rows > span:before {
+	content: counter(linenumber);
+	color: #5C6370;
+	display: block;
+	padding-right: 0.8em;
+	text-align: right;
+}

docs/how-to/deal-with-very-large-backups.md

@@ -0,0 +1,96 @@
+---
+title: How to deal with very large backups
+---
+## Biggish data
+
+Borg itself is great for efficiently de-duplicating data across successive
+backup archives, even when dealing with very large repositories. But you may
+find that while borgmatic's default mode of "prune, create, and check" works
+well on small repositories, it's not so great on larger ones. That's because
+running the default consistency checks takes a long time on large
+repositories.
+
+### A la carte actions
+
+If you find yourself in this situation, you have some options. First, you can
+run borgmatic's pruning, creating, or checking actions separately. For
+instance, the the following optional flags are available:
+
+```bash
+borgmatic --prune
+borgmatic --create
+borgmatic --check
+```
+
+You can run with only one of these flags provided, or you can mix and match
+any number of them in a single borgmatic run. This supports approaches like
+making backups with `--create` on a frequent schedule, while only running
+expensive consistency checks with `--check` on a much less frequent basis from
+a separate cron job.
+
+### Consistency check configuration
+
+Another option is to customize your consistency checks. The default
+consistency checks run both full-repository checks and per-archive checks
+within each repository.
+
+But if you find that archive checks are too slow, for example, you can
+configure borgmatic to run repository checks only. Configure this in the
+`consistency` section of borgmatic configuration:
+
+```yaml
+consistency:
+    checks:
+        - repository
+```
+
+If that's still too slow, you can disable consistency checks entirely,
+either for a single repository or for all repositories.
+
+Disabling all consistency checks looks like this:
+
+```yaml
+consistency:
+    checks:
+        - disabled
+```
+
+Or, if you have multiple repositories in your borgmatic configuration file,
+you can keep running consistency checks, but only against a subset of the
+repositories:
+
+```yaml
+consistency:
+    check_repositories:
+        - path/of/repository_to_check.borg
+```
+
+
+## Troubleshooting
+
+### Broken pipe with remote repository
+
+When running borgmatic on a large remote repository, you may receive errors
+like the following, particularly while "borg check" is validating backups for
+consistency:
+
+```text
+    Write failed: Broken pipe
+    borg: Error: Connection closed by remote host
+```
+
+This error can be caused by an ssh timeout, which you can rectify by adding
+the following to the `~/.ssh/config` file on the client:
+
+```text
+    Host *
+        ServerAliveInterval 120
+```
+
+This should make the client keep the connection alive while validating
+backups.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)

docs/how-to/develop-on-borgmatic.md

@@ -0,0 +1,112 @@
+---
+title: How to develop on borgmatic
+---
+## Source code
+
+To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
+
+```bash
+git clone https://projects.torsion.org/witten/borgmatic.git
+```
+
+Or:
+
+```bash
+git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git
+```
+
+Then, install borgmatic
+"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)"
+so that you can run borgmatic commands while you're hacking on them to
+make sure your changes work.
+
+```bash
+cd borgmatic/
+pip3 install --editable --user .
+```
+
+Note that this will typically install the borgmatic commands into
+`~/.local/bin`, which may or may not be on your PATH. There are other ways to
+install borgmatic editable as well, for instance into the system Python
+install (so without `--user`, as root), or even into a
+[virtualenv](https://virtualenv.pypa.io/en/stable/). How or where you install
+borgmatic is up to you, but generally an editable install makes development
+and testing easier.
+
+
+## Automated tests
+
+Assuming you've cloned the borgmatic source code as described above, and
+you're in the `borgmatic/` working copy, install tox, which is used for
+setting up testing environments:
+
+```bash
+pip3 install --user tox
+```
+
+Finally, to actually run tests, run:
+
+```bash
+cd borgmatic
+tox
+```
+
+### Code formatting
+
+If when running tests, you get an error from the
+[Black](https://black.readthedocs.io/en/stable/) code formatter about files
+that would be reformatted, you can ask Black to format them for you via the
+following:
+
+```bash
+tox -e black
+```
+
+And if you get a complaint from the
+[isort](https://github.com/timothycrosley/isort) Python import orderer, you
+can ask isort to order your imports for you:
+
+```bash
+tox -e isort
+```
+
+### End-to-end tests
+
+borgmatic additionally includes some end-to-end tests that integration test
+with Borg for a few representative scenarios. These tests don't run by default
+because they're relatively slow and depend on Borg. If you would like to run
+them:
+
+```bash
+tox -e end-to-end
+```
+
+## Code style
+
+Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
+the following deviations from it:
+
+ * For strings, prefer single quotes over double quotes.
+ * Limit all lines to a maximum of 100 characters.
+ * Use trailing commas within multiline values or argument lists.
+ * For multiline constructs, put opening and closing delimeters on lines
+   separate from their contents.
+ * Within multiline constructs, use standard four-space indentation. Don't align
+   indentation with an opening delimeter.
+
+borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
+formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
+the [isort](https://github.com/timothycrosley/isort) import orderer, so
+certain code style requirements will be enforced when running automated tests.
+See the Black, Flake8, and isort documentation for more information.
+
+## Continuous integration
+
+Each pull request triggers a continuous integration build which runs the test
+suite. You can view these builds on
+[build.torsion.org](https://build.torsion.org/witten/borgmatic), and they're
+also linked from the commits list on each pull request.
+
+## Related documentation
+
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)

docs/how-to/inspect-your-backups.md

@@ -0,0 +1,85 @@
+---
+title: How to inspect your backups
+---
+## Backup progress
+
+By default, borgmatic runs proceed silently except in the case of errors. But
+if you'd like to to get additional information about the progress of the
+backup as it proceeds, use the verbosity option:
+
+```bash
+borgmatic --verbosity 1
+```
+
+This lists the files that borgmatic is archiving, which are those that are new
+or changed since the last backup.
+
+Or, for even more progress and debug spew:
+
+```bash
+borgmatic --verbosity 2
+```
+
+## Existing backups
+
+Borgmatic provides convenient flags for Borg's
+[list](https://borgbackup.readthedocs.io/en/stable/usage/list.html) and
+[info](https://borgbackup.readthedocs.io/en/stable/usage/info.html)
+functionality:
+
+
+```bash
+borgmatic --list
+borgmatic --info
+```
+
+## Logging
+
+By default, borgmatic logs to a local syslog-compatible daemon if one is
+present. Where those logs show up depends on your particular system. If you're
+using systemd, try running `journalctl -xe`. Otherwise, try viewing
+`/var/log/syslog` or similiar.
+
+You can customize the log level used for syslog logging with the
+`--syslog-verbosity` flag, and this is independent from the console logging
+`--verbosity` flag described above. For instance, to get additional
+information about the progress of the backup as it proceeds:
+
+```bash
+borgmatic --syslog-verbosity 1
+```
+
+Or to increase syslog logging to include debug spew:
+
+```bash
+borgmatic --syslog-verbosity 2
+```
+
+### systemd journal
+
+If your local syslog daemon is systemd's journal, be aware that journald by
+default throttles the rate at which a particular program can log. So you may
+need to [change the journald rate
+limit](https://www.freedesktop.org/software/systemd/man/journald.conf.html#RateLimitIntervalSec=)
+in `/etc/systemd/journald.conf` if you're finding that borgmatic journald logs
+are missing.
+
+Note that the [sample borgmatic systemd service
+file](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#systemd)
+already has this rate limit disabled.
+
+## Scripting borgmatic
+
+To consume the output of borgmatic in other software, you can include an
+optional `--json` flag with `--create`, `--list`, or `--info` to get the
+output formatted as JSON.
+
+Note that when you specify the `--json` flag, Borg's other non-JSON output is
+suppressed so as not to interfere with the captured JSON. Also note that JSON
+output only shows up at the console, and not in syslog.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)

docs/how-to/make-per-application-backups.md

@@ -0,0 +1,115 @@
+---
+title: How to make per-application backups
+---
+## Multiple backup configurations
+
+You may find yourself wanting to create different backup policies for
+different applications on your system. For instance, you may want one backup
+configuration for your database data directory, and a different configuration
+for your user home directories.
+
+The way to accomplish that is pretty simple: Create multiple separate
+configuration files and place each one in a `/etc/borgmatic.d/` directory. For
+instance:
+
+```bash
+sudo mkdir /etc/borgmatic.d
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+```
+
+When you set up multiple configuration files like this, borgmatic will run
+each one in turn from a single borgmatic invocation. This includes, by
+default, the traditional `/etc/borgmatic/config.yaml` as well.
+
+And if you need even more customizability, you can specify alternate
+configuration paths on the command-line with borgmatic's `--config` option.
+See `borgmatic --help` for more information.
+
+
+## Configuration includes
+
+Once you have multiple different configuration files, you might want to share
+common configuration options across these files with having to copy and paste
+them. To achieve this, you can put fragments of common configuration options
+into a file, and then include or inline that file into one or more borgmatic
+configuration files.
+
+Let's say that you want to include common retention configuration across all
+of your configuration files. You could do that in each configuration file with
+the following:
+
+```yaml
+location:
+   ...
+
+retention:
+    !include /etc/borgmatic/common_retention.yaml
+```
+
+And then the contents of `common_retention.yaml` could be:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+To prevent borgmatic from trying to load these configuration fragments by
+themselves and complaining that they are not valid configuration files, you
+should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
+is fine.)
+
+Note that this form of include must be a YAML value rather than a key. For
+example, this will not work:
+
+```yaml
+location:
+   ...
+
+# Don't do this. It won't work!
+!include /etc/borgmatic/common_retention.yaml
+```
+
+But if you do want to merge in a YAML key and its values, keep reading!
+
+
+## Include merging
+
+If you need to get even fancier and pull in common configuration options while
+potentially overriding individual options, you can perform a YAML merge of
+included configuration using the YAML `<<` key. For instance, here's an
+example of a main configuration file that pulls in two retention options via
+an include, and then overrides one of them locally:
+
+```yaml
+location:
+   ...
+
+retention:
+    keep_daily: 5
+    <<: !include /etc/borgmatic/common_retention.yaml
+```
+
+This is what `common_retention.yaml` might look like:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration would have a
+`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
+
+When there is a collision of an option between the local file and the merged
+include, the local file's option takes precedent. And note that this is a
+shallow merge rather than a deep merge, so the merging does not descend into
+nested values.
+
+Note that this `<<` include merging syntax is only for merging in mappings
+(keys/values). If you'd like to include other types like scalars or lists
+directly, please see the section above about standard includes.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)

docs/how-to/restore-a-backup.md

@@ -0,0 +1,61 @@
+---
+title: How to restore a backup
+---
+## Extract
+
+When the worst happens—or you want to test your backups—the first step is
+to figure out which archive to restore. A good way to do that is to use the
+`--list` action:
+
+```bash
+borgmatic --list
+```
+
+That should yield output looking something like:
+
+```text
+host-2019-01-01T04:05:06.070809      Tue, 2019-01-01 04:05:06 [...]
+host-2019-01-02T04:06:07.080910      Wed, 2019-01-02 04:06:07 [...]
+```
+
+Assuming that you want to restore the archive with the most up-to-date files
+and therefore the latest timestamp, run a command like:
+
+```bash
+borgmatic --extract --archive host-2019-01-02T04:06:07.080910
+```
+
+The `--archive` value is the name of the archive to restore. This extracts the
+entire contents of the archive to the current directory, so make sure you're
+in the right place before running the command.
+
+
+## Repository selection
+
+If you have a single repository in your borgmatic configuration file(s), no
+problem: the `--extract` action figures out which repository to use.
+
+But if you have multiple repositories configured, then you'll need to specify
+the repository path containing the archive to extract. Here's an example:
+
+```bash
+borgmatic --extract --repository repo.borg --archive host-2019-...
+```
+
+## Restore particular files
+
+Sometimes, you want to restore a single deleted file, rather than restoring
+everything from an archive. To do that, tack on one or more `--restore-path`
+values. For instance:
+
+```bash
+borgmatic --extract --archive host-2019-... --restore-path /path/1 /path/2
+```
+
+Like a whole-archive restore, this also restores into the current directory.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)

docs/how-to/run-preparation-steps-before-backups.md

@@ -0,0 +1,58 @@
+---
+title: How to run preparation steps before backups
+---
+## Preparation and cleanup hooks
+
+If you find yourself performing prepraration tasks before your backup runs, or
+cleanup work afterwards, borgmatic hooks may be of interest. Hooks are
+shell commands that borgmatic executes for you at various points, and they're
+configured in the `hooks` section of your configuration file.
+
+For instance, you can specify `before_backup` hooks to dump a database to file
+before backing it up, and specify `after_backup` hooks to delete the temporary
+file afterwards. Here's an example:
+
+```yaml
+hooks:
+    before_backup:
+        - dump-a-database /to/file.sql
+    after_backup:
+        - rm /to/file.sql
+```
+
+borgmatic hooks run once per configuration file. `before_backup` hooks run
+prior to backups of all repositories. `after_backup` hooks run afterwards, but
+not if an error occurs in a previous hook or in the backups themselves.
+
+
+## Error hooks
+
+borgmatic also runs `on_error` hooks if an error occurs. Here's an example
+configuration:
+
+```yaml
+hooks:
+    on_error:
+        - echo "Error while creating a backup."
+```
+
+## Hook output
+
+Any output produced by your hooks shows up both at the console and in syslog.
+For more information, read about <a
+href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md">inspecting
+your backups</a>.
+
+## Security
+
+An important security note about hooks: borgmatic executes all hook commands
+with the user permissions of borgmatic itself. So to prevent potential shell
+injection or privilege escalation, do not forget to set secure permissions
+(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)

docs/how-to/set-up-backups.md

@@ -0,0 +1,234 @@
+---
+title: How to set up backups with borgmatic
+---
+## Installation
+
+To get up and running, first [install
+Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
+least version 1.1.
+
+Borgmatic consumes configurations in `/etc/borgmatic/` and `/etc/borgmatic.d/`
+by default. Therefore, we show how to install borgmatic for the root user which
+will have access permissions for these locations by default.
+
+Run the following commands to download and install borgmatic:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+This is a [recommended user site
+installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site).
+You will need to ensure that `/root/.local/bin` is available on your `$PATH` so
+that the borgmatic executable is available.
+
+Note that your pip binary may have a different name than "pip3". Make sure
+you're using Python 3, as borgmatic does not support Python 2.
+
+### Other ways to install
+
+Along with the above process, you have several other options for installing
+borgmatic:
+
+ * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
+ * [Docker image with support for scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
+ * [Debian](https://tracker.debian.org/pkg/borgmatic)
+ * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
+ * [Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
+ * [Arch Linux](https://aur.archlinux.org/packages/borgmatic/)
+ * [OpenBSD](http://ports.su/sysutils/borgmatic)
+ * [openSUSE](https://software.opensuse.org/package/borgmatic)
+
+
+## Hosting providers
+
+Need somewhere to store your encrypted offsite backups? The following hosting
+providers include specific support for Borg/borgmatic. Using these links and
+services helps support borgmatic development and hosting. (These are referral
+links, but without any tracking scripts or cookies.)
+
+<ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
+</ul>
+
+## Configuration
+
+After you install borgmatic, generate a sample configuration file:
+
+```bash
+sudo generate-borgmatic-config
+```
+
+If that command is not found, then it may be installed in a location that's
+not in your system `PATH`. Try looking in `/usr/local/bin/`.
+
+This generates a sample configuration file at /etc/borgmatic/config.yaml (by
+default). You should edit the file to suit your needs, as the values are
+representative. All options are optional except where indicated, so feel free
+to ignore anything you don't need.
+
+Note that the configuration file is organized into distinct sections, each
+with a section name like `location:` or `storage:`. So take care that if you
+uncomment a particular option, also uncomment its containing section name, or
+else borgmatic won't recognize the option.
+
+You can also get the same sample configuration file from the [configuration
+reference](https://torsion.org/borgmatic/docs/reference/configuration.md), the authoritative set of
+all configuration options. This is handy if borgmatic has added new options
+since you originally created your configuration file.
+
+
+### Encryption
+
+Note that if you plan to run borgmatic on a schedule with cron, and you
+encrypt your Borg repository with a passphrase instead of a key file, you'll
+either need to set the borgmatic `encryption_passphrase` configuration
+variable or set the `BORG_PASSPHRASE` environment variable. See the
+[repository encryption
+section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
+of the Borg Quick Start for more info.
+
+Alternatively, you can specify the passphrase programatically by setting
+either the borgmatic `encryption_passcommand` configuration variable or the
+`BORG_PASSCOMMAND` environment variable. See the [Borg Security
+FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
+for more info.
+
+
+### Validation
+
+If you'd like to validate that your borgmatic configuration is valid, the
+following command is available for that:
+
+```bash
+sudo validate-borgmatic-config
+```
+
+This command's exit status (`$?` in Bash) is zero when configuration is valid
+and non-zero otherwise.
+
+Validating configuration can be useful if you generate your configuration
+files via configuration management, or you want to double check that your hand
+edits are valid.
+
+
+## Initialization
+
+Before you can create backups with borgmatic, you first need to initialize a
+Borg repository so you have a destination for your backup archives. (But skip
+this step if you already have a Borg repository.) To create a repository, run
+a command like the following:
+
+```bash
+borgmatic --init --encryption repokey
+```
+
+This uses the borgmatic configuration file you created above to determine
+which local or remote repository to create, and encrypts it with the
+encryption passphrase specified there if one is provided. Read about [Borg
+encryption
+modes](https://borgbackup.readthedocs.io/en/latest/usage/init.html#encryption-modes)
+for the menu of available encryption modes.
+
+Also, optionally check out the [Borg Quick
+Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) for more
+background about repository initialization.
+
+Note that borgmatic skips repository initialization if the repository already
+exists. This supports use cases like ensuring a repository exists prior to
+performing a backup.
+
+If the repository is on a remote host, make sure that your local user has
+key-based SSH access to the desired user account on the remote host.
+
+
+## Backups
+
+Now that you've configured borgmatic and initialized a repository, it's a
+good idea to test that borgmatic is working. So to run borgmatic and start a
+backup, you can invoke it like this:
+
+```bash
+borgmatic --verbosity 1
+```
+
+By default, this will also prune any old backups as per the configured
+retention policy, and check backups for consistency problems due to things
+like file damage.
+
+The verbosity flag makes borgmatic list the files that it's archiving, which
+are those that are new or changed since the last backup. Eyeball the list and
+see if it matches your expectations based on the configuration.
+
+
+## Autopilot
+
+Running backups manually is good for validating your configuration, but I'm
+guessing that you want to run borgmatic automatically, say once a day. To do
+that, you can configure a separate job runner to invoke it periodically.
+
+### cron
+
+If you're using cron, download the [sample cron
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic).
+Then, from the directory where you downloaded it:
+
+```bash
+sudo mv borgmatic /etc/cron.d/borgmatic
+sudo chmod +x /etc/cron.d/borgmatic
+```
+
+You can modify the cron file if you'd like to run borgmatic more or less frequently.
+
+### systemd
+
+If you're using systemd instead of cron to run jobs, download the [sample
+systemd service
+file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
+and the [sample systemd timer
+file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
+Then, from the directory where you downloaded them:
+
+```bash
+sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
+sudo systemctl enable borgmatic.timer
+sudo systemctl start borgmatic.timer
+```
+
+Feel free to modify the timer file based on how frequently you'd like
+borgmatic to run.
+
+## Colored output
+
+Borgmatic produces colored terminal output by default. It is disabled when a
+non-interactive terminal is detected (like a cron job). Otherwise, you can
+disable it by passing the `--no-color` flag, setting the environment variable
+`PY_COLORS=False`, or setting the `color` option to `false` in the `output`
+section of configuration.
+
+## Troubleshooting
+
+### libyaml compilation errors
+
+borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
+use a C YAML library (libyaml) if present. But if it's not installed, then
+when installing or upgrading borgmatic, you may see errors about compiling the
+YAML library. If so, not to worry. borgmatic should install and function
+correctly even without the C YAML library. And borgmatic won't be any faster
+with the C library present, so you don't need to go out of your way to install
+it.
+
+
+## Related documentation
+
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
+ * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration.md)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line.md)
+
+<script>
+  var links = document.getElementsByClassName("referral");
+  links[Math.floor(Math.random() * links.length)].style.display = "none";
+</script>

docs/how-to/upgrade.md

@@ -0,0 +1,79 @@
+---
+title: How to upgrade borgmatic
+---
+## Upgrading
+
+In general, all you should need to do to upgrade borgmatic is run the
+following:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+See below about special cases.
+
+
+### Upgrading from borgmatic 1.0.x
+
+borgmatic changed its configuration file format in version 1.1.0 from
+INI-style to YAML. This better supports validation, and has a more natural way
+to express lists of values. To upgrade your existing configuration, first
+upgrade to the new version of borgmatic.
+
+As of version 1.1.0, borgmatic no longer supports Python 2. If you were
+already running borgmatic with Python 3, then you can upgrade borgmatic
+in-place:
+
+```bash
+sudo pip3 install --user --upgrade borgmatic
+```
+
+But if you were running borgmatic with Python 2, uninstall and reinstall instead:
+
+```bash
+sudo pip uninstall borgmatic
+sudo pip3 install --user borgmatic
+```
+
+The pip binary names for different versions of Python can differ, so the above
+commands may need some tweaking to work on your machine.
+
+
+Once borgmatic is upgraded, run:
+
+```bash
+sudo upgrade-borgmatic-config
+```
+
+That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
+(by default) using the values from both your existing configuration and
+excludes files. The new version of borgmatic will consume the YAML
+configuration file instead of the old one.
+
+
+### Upgrading from atticmatic
+
+You can ignore this section if you're not an atticmatic user (the former name
+of borgmatic).
+
+borgmatic only supports Borg now and no longer supports Attic. So if you're
+an Attic user, consider switching to Borg. See the [Borg upgrade
+command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
+for more information. Then, follow the instructions above about setting up
+your borgmatic configuration files.
+
+If you were already using Borg with atticmatic, then you can upgrade
+from atticmatic to borgmatic by running the following commands:
+
+```bash
+sudo pip3 uninstall atticmatic
+sudo pip3 install --user borgmatic
+```
+
+That's it! borgmatic will continue using your /etc/borgmatic configuration
+files.
+
+
+## Related documentation
+
+ * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic.md)

docs/reference/command-line.md

@@ -0,0 +1,16 @@
+---
+title: borgmatic command-line reference
+---
+## borgmatic options
+
+Here are all of the available borgmatic command-line options:
+
+```
+{% include borgmatic/command-line.txt %}
+```
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration.md)

docs/reference/configuration.md

@@ -0,0 +1,19 @@
+---
+title: borgmatic configuration reference
+---
+## Configuration file
+
+Here is a full sample borgmatic configuration file including all available options: 
+
+```yaml
+{% include borgmatic/config.yaml %}
+```
+
+Note that you can also [download this configuration
+file](https://torsion.org/borgmatic/docs/reference/config.yaml) for use locally.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line.md)

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 100
+skip-string-normalization = true

sample/cron-alpine/borgmatic

@@ -0,0 +1,3 @@
+# You can drop this file into /etc/cron.d/ to run borgmatic nightly.
+
+0 3 * * * PATH=$PATH:/usr/bin /root/.local/bin/borgmatic

sample/cron/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/borgmatic
+0 3 * * * root PATH=$PATH:/usr/local/bin /root/.local/bin/borgmatic

sample/systemd/borgmatic.service

@@ -3,4 +3,5 @@ Description=borgmatic backup
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/borgmatic
+ExecStart=/root/.local/bin/borgmatic
+LogRateLimitIntervalSec=0

sample/systemd/borgmatic.timer

@@ -3,6 +3,7 @@ Description=Run borgmatic backup
 
 [Timer]
 OnCalendar=daily
+Persistent=true
 
 [Install]
 WantedBy=timers.target

scripts/dev-docs

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+docker build --tag borgmatic-docs --file docs/Dockerfile .
+docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs

scripts/find-unsupported-borg-options

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -o nounset
+
+# For each Borg sub-command that borgmatic uses, print out the Borg flags that borgmatic does not
+# appear to support yet. This script isn't terribly robust. It's intended as a basic tool to ferret
+# out unsupported Borg options so that they can be considered for addition to borgmatic.
+
+# Generate a sample borgmatic configuration with all options set, and uncomment all options.
+generate-borgmatic-config --destination temp.yaml
+cat temp.yaml | sed -e 's/# \S.*$//' | sed -e 's/#//' > temp.yaml.uncommented
+mv temp.yaml.uncommented temp.yaml
+
+# For each sub-command (prune, create, and check), collect the Borg command-line flags that result
+# from running borgmatic with the generated configuration. Then, collect the full set of available
+# Borg flags as reported by "borg --help" for that sub-command. Finally, compare the two lists of
+# flags to determine which Borg flags borgmatic doesn't yet support.
+for sub_command in prune create check list info; do
+    echo "********** borg $sub_command **********"
+
+    for line in $(borgmatic --config temp.yaml --$sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
+        echo "$line" | grep '^-' >> borgmatic_borg_flags
+    done
+    sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
+    mv borgmatic_borg_flags.sorted borgmatic_borg_flags
+
+    for word in $(borg $sub_command --help | grep '^  -') ; do
+        # Exclude a bunch of flags that borgmatic actually supports, but don't get exercised by the
+        # generated sample config, and also flags that don't make sense to support.
+        echo "$word" | grep ^-- | sed -e 's/,$//' \
+            | grep -v '^--archives-only$' \
+            | grep -v '^--critical$' \
+            | grep -v '^--debug$' \
+            | grep -v '^--dry-run$' \
+            | grep -v '^--error$' \
+            | grep -v '^--help$' \
+            | grep -v '^--info$' \
+            | grep -v '^--json$' \
+            | grep -v '^--keep-last$' \
+            | grep -v '^--list$' \
+            | grep -v '^--nobsdflags$' \
+            | grep -v '^--pattern$' \
+            | grep -v '^--progress$' \
+            | grep -v '^--stats$' \
+            | grep -v '^--read-special$' \
+            | grep -v '^--repository-only$' \
+            | grep -v '^--show-rc$' \
+            | grep -v '^--stats$' \
+            | grep -v '^--verbose$' \
+            | grep -v '^--warning$' \
+            | grep -v '^-h$' \
+            >> all_borg_flags
+    done
+
+    sort all_borg_flags > all_borg_flags.sorted
+    mv all_borg_flags.sorted all_borg_flags
+
+    comm -13 borgmatic_borg_flags all_borg_flags
+
+    rm ./*_borg_flags
+done
+
+rm temp.yaml

scripts/pip

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+# Temporary work around for https://github.com/pypa/pip/issues/6434
+python -m pip install --upgrade pip==19.1.1
+python -m pip install --no-use-pep517 $*
+

scripts/push

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+branch_name=$(git rev-parse --abbrev-ref HEAD)
+
+git push -u github "$branch_name"
+git push -u origin "$branch_name"

scripts/release

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+projects_token=${1:-}
+github_token=${2:-}
+
+if [[ -z $github_token ]]; then
+    echo "Usage: $0 [projects-token] [github-token]"
+    exit 1
+fi
+if [[ ! -f NEWS ]]; then
+    echo "Missing NEWS file. Try running from root of repository."
+    exit 1
+fi
+
+version=$(head --lines=1 NEWS)
+git tag $version
+git push origin $version
+git push github $version
+
+# Build borgmatic and publish to pypi.
+rm -fr dist
+python3 setup.py bdist_wheel
+python3 setup.py sdist
+twine upload -r pypi dist/borgmatic-*.tar.gz
+twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
+
+# Set release changelogs on projects.evoworx.org and GitHub.
+release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
+escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
+curl --silent --request POST \
+    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \
+    --header "Accept: application/json" \
+    --header "Content-Type: application/json" \
+    --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}"
+
+github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" \
+    --name="borgmatic $version" --body="$release_changelog"

scripts/run-tests

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# This script is intended to be run from the continuous integration build
+# server, and not on a developer machine. For that, see:
+# https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
+
+set -e
+
+python -m pip install --upgrade pip==19.1.1
+pip install tox==3.10.0
+tox
+apk add --no-cache borgbackup
+tox -e end-to-end

setup.cfg

@@ -1,2 +1,20 @@
 [metadata]
 description-file=README.md
+
+[tool:pytest]
+testpaths = tests
+addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end
+filterwarnings = 
+    ignore:Coverage disabled.*:pytest.PytestWarning
+
+[flake8]
+ignore = E501,W503
+exclude = *.*/*
+
+[tool:isort]
+force_single_line = False
+include_trailing_comma = True
+known_first_party = borgmatic
+line_length = 100
+multi_line_output = 3
+skip = .tox

setup.py

@@ -1,18 +1,16 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-
-VERSION = '1.1.7'
+VERSION = '1.3.8'
 
 
 setup(
     name='borgmatic',
     version=VERSION,
-    description='A wrapper script for Borg backup software that creates and prunes backups',
+    description='Simple, configuration-driven backup software for servers and workstations',
     author='Dan Helfman',
     author_email='witten@torsion.org',
     url='https://torsion.org/borgmatic',
-    download_url='https://torsion.org/hg/borgmatic/archive/%s.tar.gz' % VERSION,
-    classifiers=(
+    classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
         'Intended Audience :: System Administrators',
@@ -20,26 +18,22 @@ setup(
         'Programming Language :: Python',
         'Topic :: Security :: Cryptography',
         'Topic :: System :: Archiving :: Backup',
-    ),
-    packages=find_packages(),
+    ],
+    packages=find_packages(exclude=['tests*']),
     entry_points={
         'console_scripts': [
             'borgmatic = borgmatic.commands.borgmatic:main',
             'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
             'generate-borgmatic-config = borgmatic.commands.generate_config:main',
+            'validate-borgmatic-config = borgmatic.commands.validate_config:main',
         ]
     },
-    obsoletes=[
-        'atticmatic',
-    ],
+    obsoletes=['atticmatic'],
     install_requires=(
-        'pykwalify',
-        'ruamel.yaml<=0.15',
+        'pykwalify>=1.6.0,<14.06',
+        'ruamel.yaml>0.15.0,<0.16.0',
         'setuptools',
-    ),
-    tests_require=(
-        'flexmock',
-        'pytest',
+        'colorama>=0.4.1,<0.5',
     ),
     include_package_data=True,
 )

test_requirements.txt

@@ -1,5 +1,24 @@
-flexmock==0.10.2
-pykwalify==1.6.0
-pytest==2.9.1
-pytest-cov==2.5.1
-ruamel.yaml==0.15.18
+appdirs==1.4.3
+atomicwrites==1.3.0
+attrs==19.1.0
+black==19.3b0; python_version >= '3.6'
+click==7.0
+colorama==0.4.1
+coverage==4.5.3
+docopt==0.6.2
+flake8==3.7.7
+flexmock==0.10.4
+isort==4.3.20
+mccabe==0.6.1
+more-itertools==7.0.0
+pluggy==0.12.0
+py==1.8.0
+pycodestyle==2.5.0
+pyflakes==2.1.1
+pykwalify==1.7.0
+pytest==4.6.3
+pytest-cov==2.7.1
+python-dateutil==2.8.0
+PyYAML==5.1.1
+ruamel.yaml>0.15.0,<0.16.0
+toml==0.10.0

tests/end-to-end/test_borgmatic.py

@@ -0,0 +1,81 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing (including injecting the given repository path and tacking on an encryption
+    passphrase).
+    '''
+    subprocess.check_call(
+        'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+    )
+    config = (
+        open(config_path)
+        .read()
+        .replace('user@backupserver:sourcehostname.borg', repository_path)
+        .replace('- /home', '- {}'.format(config_path))
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + 'storage:\n    encryption_passphrase: "test"'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_borgmatic_command():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    extract_path = os.path.join(temporary_directory, 'extract')
+
+    original_working_directory = os.getcwd()
+    os.mkdir(extract_path)
+    os.chdir(extract_path)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path)
+
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} --init --encryption repokey'.format(config_path).split(' ')
+        )
+
+        # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
+        subprocess.check_call('borgmatic --config {}'.format(config_path).split(' '))
+        output = subprocess.check_output(
+            'borgmatic --config {} --list --json'.format(config_path).split(' ')
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
+
+        # Extract the created archive into the current (temporary) directory, and confirm that the
+        # extracted file looks right.
+        output = subprocess.check_output(
+            'borgmatic --config {} --extract --archive {}'.format(config_path, archive_name).split(
+                ' '
+            )
+        ).decode(sys.stdout.encoding)
+        extracted_config_path = os.path.join(extract_path, config_path)
+        assert open(extracted_config_path).read() == open(config_path).read()
+
+        # Exercise the info flag.
+        output = subprocess.check_output(
+            'borgmatic --config {} --info --json'.format(config_path).split(' ')
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert 'repository' in parsed_output[0]
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/end-to-end/test_validate_config.py

@@ -0,0 +1,36 @@
+import os
+import subprocess
+import tempfile
+
+
+def test_validate_config_command_with_valid_configuration_succeeds():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 0
+
+
+def test_validate_config_command_with_invalid_configuration_fails():
+    with tempfile.TemporaryDirectory() as temporary_directory:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+
+        subprocess.check_call(
+            'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+        )
+        config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"')
+        config_file = open(config_path, 'w')
+        config_file.write(config)
+        config_file.close()
+
+        exit_code = subprocess.call(
+            'validate-borgmatic-config --config {}'.format(config_path).split(' ')
+        )
+
+        assert exit_code == 1

tests/integration/commands/test_borgmatic.py

@@ -0,0 +1,285 @@
+import subprocess
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.commands import borgmatic as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_paths == config_paths
+    assert parser.excludes_filename is None
+    assert parser.verbosity == 0
+    assert parser.syslog_verbosity == 0
+    assert parser.json is False
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    assert parser.config_paths == ['myconfig', 'otherconfig']
+    assert parser.verbosity == 0
+    assert parser.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments('--verbosity', '1')
+
+    assert parser.config_paths == config_paths
+    assert parser.excludes_filename is None
+    assert parser.verbosity == 1
+    assert parser.syslog_verbosity == 0
+
+
+def test_parse_arguments_with_syslog_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments('--syslog-verbosity', '2')
+
+    assert parser.config_paths == config_paths
+    assert parser.excludes_filename is None
+    assert parser.verbosity == 0
+    assert parser.syslog_verbosity == 2
+
+
+def test_parse_arguments_with_json_overrides_default():
+    parser = module.parse_arguments('--list', '--json')
+    assert parser.json is True
+
+
+def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments()
+
+    assert parser.prune is True
+    assert parser.create is True
+    assert parser.check is True
+
+
+def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--prune')
+
+    assert parser.prune is True
+    assert parser.create is False
+    assert parser.check is False
+
+
+def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--create', '--check')
+
+    assert parser.prune is False
+    assert parser.create is True
+    assert parser.check is True
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')
+
+
+def test_parse_arguments_disallows_deprecated_excludes_option():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+
+def test_parse_arguments_disallows_encryption_mode_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey')
+
+
+def test_parse_arguments_allows_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey')
+
+
+def test_parse_arguments_requires_encryption_mode_with_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--init')
+
+
+def test_parse_arguments_disallows_append_only_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--append-only')
+
+
+def test_parse_arguments_disallows_storage_quota_without_init():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G')
+
+
+def test_parse_arguments_allows_init_and_prune():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--prune')
+
+
+def test_parse_arguments_allows_init_and_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--init', '--encryption', 'repokey', '--create')
+
+
+def test_parse_arguments_disallows_init_and_dry_run():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments(
+            '--config', 'myconfig', '--init', '--encryption', 'repokey', '--dry-run'
+        )
+
+
+def test_parse_arguments_disallows_repository_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
+
+
+def test_parse_arguments_allows_repository_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments(
+        '--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
+    )
+
+
+def test_parse_arguments_allows_repository_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
+
+
+def test_parse_arguments_disallows_archive_without_extract_or_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--archive', 'test')
+
+
+def test_parse_arguments_disallows_restore_paths_without_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--restore-path', 'test')
+
+
+def test_parse_arguments_allows_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
+
+
+def test_parse_arguments_allows_archive_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
+
+
+def test_parse_arguments_requires_archive_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--config', 'myconfig', '--extract')
+
+
+def test_parse_arguments_allows_progress_and_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', '--create', '--list')
+
+
+def test_parse_arguments_allows_progress_and_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
+
+
+def test_parse_arguments_disallows_progress_without_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--progress', '--list')
+
+
+def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', '--create', '--list')
+
+
+def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats', '--prune', '--list')
+
+
+def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--stats', '--list')
+
+
+def test_parse_arguments_with_just_stats_flag_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--stats')
+
+
+def test_parse_arguments_allows_json_with_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--list', '--json')
+    module.parse_arguments('--info', '--json')
+
+
+def test_parse_arguments_disallows_json_without_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--json')
+
+
+def test_parse_arguments_disallows_json_with_both_list_and_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('--list', '--info', '--json')
+
+
+def test_borgmatic_version_matches_news_version():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
+    news_version = open('NEWS').readline()
+
+    assert borgmatic_version == news_version

tests/integration/commands/test_convert_config.py

@@ -1,7 +1,7 @@
 import os
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.commands import convert_config as module
 
@@ -20,9 +20,12 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
     flexmock(os.path).should_receive('exists').and_return(True)
 
     parser = module.parse_arguments(
-        '--source-config', 'config',
-        '--source-excludes', 'excludes',
-        '--destination-config', 'config.yaml',
+        '--source-config',
+        'config',
+        '--source-excludes',
+        'excludes',
+        '--destination-config',
+        'config.yaml',
     )
 
     assert parser.source_config_filename == 'config'

tests/integration/commands/test_generate_config.py

@@ -6,6 +6,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
 
     assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
 
+
 def test_parse_arguments_with_filename_argument_overrides_defaults():
     parser = module.parse_arguments('--destination', 'config.yaml')
 

tests/integration/commands/test_validate_config.py

@@ -0,0 +1,20 @@
+from flexmock import flexmock
+
+from borgmatic.commands import validate_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_paths == config_paths
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    assert parser.config_paths == ['myconfig', 'otherconfig']

tests/integration/config/test_generate.py

@@ -1,9 +1,9 @@
-from io import StringIO
 import os
 import sys
+from io import StringIO
 
-from flexmock import flexmock
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import generate as module
 
@@ -11,11 +11,74 @@ from borgmatic.config import generate as module
 def test_insert_newline_before_comment_does_not_raise():
     field_name = 'foo'
     config = module.yaml.comments.CommentedMap([(field_name, 33)])
-    config.yaml_set_comment_before_after_key(key=field_name, before='Comment',)
+    config.yaml_set_comment_before_after_key(key=field_name, before='Comment')
 
     module._insert_newline_before_comment(config, field_name)
 
 
+def test_comment_out_line_skips_blank_line():
+    line = '    \n'
+
+    assert module._comment_out_line(line) == line
+
+
+def test_comment_out_line_skips_already_commented_out_line():
+    line = '    # foo'
+
+    assert module._comment_out_line(line) == line
+
+
+def test_comment_out_line_comments_section_name():
+    line = 'figgy-pudding:'
+
+    assert module._comment_out_line(line) == '#' + line
+
+
+def test_comment_out_line_comments_indented_option():
+    line = '    enabled: true'
+
+    assert module._comment_out_line(line) == '    #enabled: true'
+
+
+def test_comment_out_optional_configuration_comments_optional_config_only():
+    flexmock(module)._comment_out_line = lambda line: '#' + line
+    config = '''
+foo:
+    bar:
+        - baz
+        - quux
+
+location:
+    repositories:
+        - one
+        - two
+
+    other: thing
+    '''
+
+    expected_config = '''
+#foo:
+#    bar:
+#        - baz
+#        - quux
+#
+location:
+    repositories:
+        - one
+        - two
+#
+#    other: thing
+    '''
+
+    assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
+
+
+def test_render_configuration_does_not_raise():
+    flexmock(module.yaml).should_receive('round_trip_dump')
+
+    module._render_configuration({})
+
+
 def test_write_configuration_does_not_raise():
     flexmock(os.path).should_receive('exists').and_return(False)
     flexmock(os).should_receive('makedirs')
@@ -23,14 +86,14 @@ def test_write_configuration_does_not_raise():
     builtins.should_receive('open').and_return(StringIO())
     flexmock(os).should_receive('chmod')
 
-    module.write_configuration('config.yaml', {})
+    module.write_configuration('config.yaml', 'config: yaml')
 
 
 def test_write_configuration_with_already_existing_file_raises():
     flexmock(os.path).should_receive('exists').and_return(True)
 
     with pytest.raises(FileExistsError):
-        module.write_configuration('config.yaml', {})
+        module.write_configuration('config.yaml', 'config: yaml')
 
 
 def test_write_configuration_with_already_existing_directory_does_not_raise():
@@ -40,18 +103,13 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
     builtins.should_receive('open').and_return(StringIO())
     flexmock(os).should_receive('chmod')
 
-    module.write_configuration('config.yaml', {})
+    module.write_configuration('config.yaml', 'config: yaml')
 
 
 def test_add_comments_to_configuration_does_not_raise():
     # Ensure that it can deal with fields both in the schema and missing from the schema.
     config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
-    schema = {
-        'map': {
-            'foo': {'desc': 'Foo'},
-            'bar': {'desc': 'Bar'},
-        }
-    }
+    schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
 
     module.add_comments_to_configuration(config, schema)
 
@@ -59,7 +117,9 @@ def test_add_comments_to_configuration_does_not_raise():
 def test_generate_sample_configuration_does_not_raise():
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('schema.yaml').and_return('')
-    flexmock(module).should_receive('write_configuration')
     flexmock(module).should_receive('_schema_to_sample_configuration')
+    flexmock(module).should_receive('_render_configuration')
+    flexmock(module).should_receive('_comment_out_optional_configuration')
+    flexmock(module).should_receive('write_configuration')
 
     module.generate_sample_configuration('config.yaml', 'schema.yaml')

tests/integration/config/test_legacy.py

@@ -1,24 +1,18 @@
-from io import StringIO
-
-from collections import OrderedDict
 import string
+from collections import OrderedDict
+from io import StringIO
 
 from borgmatic.config import legacy as module
 
 
 def test_parse_section_options_with_punctuation_should_return_section_options():
     parser = module.RawConfigParser()
-    parser.readfp(StringIO('[section]\nfoo: {}\n'.format(string.punctuation)))
+    parser.read_file(StringIO('[section]\nfoo: {}\n'.format(string.punctuation)))
 
     section_format = module.Section_format(
-        'section',
-        (module.Config_option('foo', str, required=True),),
+        'section', (module.Config_option('foo', str, required=True),)
     )
 
     config = module.parse_section_options(parser, section_format)
 
-    assert config == OrderedDict(
-        (
-            ('foo', string.punctuation),
-        )
-    )
+    assert config == OrderedDict((('foo', string.punctuation),))

tests/integration/config/test_load.py

@@ -0,0 +1,62 @@
+import sys
+
+import pytest
+import ruamel.yaml
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_load_configuration_parses_contents():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_inlines_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return('value')
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        'key: !include include.yaml'
+    )
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_merges_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        foo: bar
+        baz: quux
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: override
+        <<: !include include.yaml
+        '''
+    )
+
+    assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
+
+
+def test_load_configuration_does_not_merge_include_list():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+          - one
+          - two
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: bar
+        repositories:
+          <<: !include include.yaml
+        '''
+    )
+
+    with pytest.raises(ruamel.yaml.error.YAMLError):
+        assert module.load_configuration('config.yaml')

tests/integration/config/test_validate.py

@@ -0,0 +1,214 @@
+import io
+import string
+import sys
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.config import validate as module
+
+
+def test_schema_filename_returns_plausable_path():
+    schema_path = module.schema_filename()
+
+    assert schema_path.endswith('/schema.yaml')
+
+
+def mock_config_and_schema(config_yaml, schema_yaml=None):
+    '''
+    Set up mocks for the given config config YAML string and the schema YAML string, or the default
+    schema if no schema is provided. The idea is that that the code under test consumes these mocks
+    when parsing the configuration.
+    '''
+    config_stream = io.StringIO(config_yaml)
+    if schema_yaml is None:
+        schema_stream = open(module.schema_filename())
+    else:
+        schema_stream = io.StringIO(schema_yaml)
+
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_stream)
+    builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream)
+
+
+def test_parse_configuration_transforms_file_into_mapping():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+                - /etc
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_minutely: 60
+            keep_hourly: 24
+            keep_daily: 7
+
+        consistency:
+            checks:
+                - repository
+                - archives
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60},
+        'consistency': {'checks': ['repository', 'archives']},
+    }
+
+
+def test_parse_configuration_passes_through_quoted_punctuation():
+    escaped_punctuation = string.punctuation.replace('\\', r'\\').replace('"', r'\"')
+
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - "{}.borg"
+        '''.format(
+            escaped_punctuation
+        )
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {
+            'source_directories': ['/home'],
+            'repositories': ['{}.borg'.format(string.punctuation)],
+        }
+    }
+
+
+def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+        ''',
+        '''
+        map:
+            location:
+                required: true
+                map:
+                    source_directories:
+                        required: true
+                        seq:
+                            - type: scalar
+                    repositories:
+                        required: true
+                        seq:
+                            - type: scalar
+        ''',
+    )
+
+    module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_inlines_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24},
+    }
+
+
+def test_parse_configuration_merges_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_daily: 1
+            <<: !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 1, 'keep_hourly': 24},
+    }
+
+
+def test_parse_configuration_raises_for_missing_config_file():
+    with pytest.raises(FileNotFoundError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_missing_schema_file():
+    mock_config_and_schema('')
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)
+
+    with pytest.raises(FileNotFoundError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_syntax_error():
+    mock_config_and_schema('foo:\nbar')
+
+    with pytest.raises(ValueError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_validation_error():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories: yes
+            repositories:
+                - hostname.borg
+        '''
+    )
+
+    with pytest.raises(module.Validation_error):
+        module.parse_configuration('config.yaml', 'schema.yaml')

tests/integration/test_execute.py

@@ -0,0 +1,36 @@
+import logging
+import subprocess
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic import execute as module
+
+
+def test_execute_and_log_output_logs_each_line_separately():
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
+
+    module.execute_and_log_output(['echo', 'hi'], output_log_level=logging.INFO, shell=False)
+    module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False)
+
+
+def test_execute_and_log_output_logs_borg_error_as_error():
+    flexmock(module.logger).should_receive('error').with_args('borg: error: oopsie').once()
+
+    module.execute_and_log_output(
+        ['echo', 'borg: error: oopsie'], output_log_level=logging.INFO, shell=False
+    )
+
+
+def test_execute_and_log_output_with_no_output_logs_nothing():
+    flexmock(module.logger).should_receive('log').never()
+
+    module.execute_and_log_output(['true'], output_log_level=logging.INFO, shell=False)
+
+
+def test_execute_and_log_output_with_error_exit_status_raises():
+    flexmock(module.logger).should_receive('log').never()
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)