Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52 (#85).

Thanks! This will go out in the next release.

.gitignore

@@ -1,6 +1,9 @@
 *.egg-info
 *.pyc
 *.swp
+.cache
+.coverage
+.pytest_cache
 .tox
 build
 dist

.hgignore

@@ -1,7 +0,0 @@
-syntax: glob
-*.egg-info
-*.pyc
-*.swp
-.tox
-build
-dist

.hgtags

@@ -1,28 +0,0 @@
-467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
-7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
-4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-569aef47a9b25c55b13753f94706f5d330219995 0.0.5
-a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
-7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
-cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
-38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
-7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
-83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
-acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
-e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
-0afff209b902698c2266986129d6dc9f5f913101 0.1.5
-4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
-5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
-977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
-0000000000000000000000000000000000000000 github/yaml_config_files
-28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master
-dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8

AUTHORS

@@ -2,4 +2,10 @@ Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
 Henning Schroeder: Copy editing
+Johannes Feichtner: Support for user hooks
+Michele Lazzeri: Custom archive names
+newtonne: Read encryption password from external file
 Robin `ypid` Schneider: Support additional options of Borg
+Scott Squires: Custom archive names
+Thomas LÉVEIL: Support for a keep_minutely prune option. Support for the --json option
+Nick Whyte: Support prefix filtering for archive consistency checks

MANIFEST.in

@@ -0,0 +1 @@
+include borgmatic/config/schema.yaml

NEWS

@@ -1,3 +1,151 @@
+1.2.2
+ * #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in
+   borgmatic as a pykwalify RuleError.
+
+1.2.1
+ * Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info.
+ * #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/
+   path.
+ * #74, #83: Support for Borg --json option via borgmatic command-line to --list archives or show
+   archive --info in JSON format, ideal for programmatic consumption.
+ * #38, #76: Upgrade ruamel.yaml compatibility version range and fix support for Python 3.7.
+ * #77: Skip non-"*.yaml" config filenames in /etc/borgmatic.d/ so as not to parse backup files,
+   editor swap files, etc.
+ * #81: Document user-defined hooks run before/after backup, or on error.
+ * Add code style guidelines to the documention.
+
+1.2.0
+ * #61: Support for Borg --list option via borgmatic command-line to list all archives.
+ * #61: Support for Borg --info option via borgmatic command-line to display summary information.
+ * #62: Update README to mention other ways of installing borgmatic.
+ * Support for Borg --prefix option for consistency checks via "prefix" option in borgmatic's
+   consistency configuration.
+ * Add introductory screencast link to documentation.
+ * #59: Ignore "check_last" and consistency "prefix" when "archives" not in consistency checks.
+ * #60: Add "Persistent" flag to systemd timer example.
+ * #63: Support for Borg --nobsdflags option to skip recording bsdflags (e.g. NODUMP, IMMUTABLE) in
+   archive.
+ * #69: Support for Borg prune --umask option using value of existing "umask" option in borgmatic's
+   storage configuration.
+ * Update tox.ini to only assume Python 3.x instead of Python 3.4 specifically.
+ * Add ~/.config/borgmatic/config.yaml to default configuration path probing.
+ * Document how to develop on and contribute to borgmatic.
+
+1.1.15
+ * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
+ * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
+   Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
+ * #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
+ * #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
+ * #58: Support for using tilde in exclude_patterns to reference home directory.
+
+1.1.14
+ * #49: Fix for typo in --patterns-from option.
+ * #47: Support for Borg --dry-run option via borgmatic command-line.
+
+1.1.13
+ * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
+   "archives", and "extract") are specified in borgmatic configuration.
+ * #48: Add "local_path" to configuration for specifying an alternative Borg executable path.
+ * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
+   includes/excludes.
+ * Moved issue tracker from Taiga to integrated Gitea tracker at
+   https://projects.torsion.org/witten/borgmatic/issues
+
+1.1.12
+ * #46: Declare dependency on pykwalify 1.6 or above, as older versions yield "Unknown key: version"
+   rule errors.
+ * Support for Borg --keep-minutely prune option.
+
+1.1.11
+ * #26: Add "ssh_command" to configuration for specifying a custom SSH command or options.
+ * Fix for incorrect /etc/borgmatic.d/ configuration path probing on macOS. This problem manifested
+   as an error on startup: "[Errno 2] No such file or directory: '/etc/borgmatic.d'".
+
+1.1.10
+ * Pass several Unix signals through to child processes like Borg. This means that Borg now properly
+   shuts down if borgmatic is terminated (e.g. due to a system suspend).
+ * #30: Support for using tilde in repository paths to reference home directory.
+ * #43: Support for Borg --files-cache option for setting the files cache operation mode.
+ * #45: Support for Borg --remote-ratelimit option for limiting upload rate.
+ * Log invoked Borg commands when at highest verbosity level.
+
+1.1.9
+ * #17, #39: Support for user-defined hooks before/after backup, or on error.
+ * #34: Improve clarity of logging spew at high verbosity levels.
+ * #30: Support for using tilde in source directory path to reference home directory.
+ * Require "prefix" in retention section when "archive_name_format" is set. This is to avoid
+   accidental pruning of archives with a different archive name format. For similar reasons, default
+   "prefix" to "{hostname}-" if not specified.
+ * Convert main source repository from Mercurial to Git.
+ * Update dead links to Borg documentation.
+
+1.1.8
+ * #40: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default
+   config paths.
+
+1.1.7
+
+ * #29: Add "archive_name_format" to configuration for customizing archive names.
+ * Fix for traceback when "exclude_from" value is empty in configuration file.
+ * When pruning, make highest verbosity level list archives kept and pruned.
+ * Clarification of Python 3 pip usage in documentation.
+
+1.1.6
+
+ * #13, #36: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options.
+
+1.1.5
+
+ * #35: New "extract" consistency check that performs a dry-run extraction of the most recent
+   archive.
+
+1.1.4
+
+ * #18: Added command-line flags for performing a borgmatic run with only pruning, creating, or
+   checking enabled. This supports use cases like running consistency checks from a different cron
+   job with a different frequency, or running pruning with a different verbosity level.
+
+1.1.3
+
+ * #15: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
+ * Fix for generate-borgmatic-config writing config with invalid one_file_system value.
+
+1.1.2
+
+ * #33: Fix for passing check_last as integer to subprocess when calling Borg.
+
+1.1.1
+
+ * Part of #33: Fix for upgrade-borgmatic-config converting check_last option as a string instead of
+   an integer.
+ * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present.
+
+1.1.0
+
+ * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade.
+ * Added generate-borgmatic-config command for initial config creation.
+ * Dropped Python 2 support. Now Python 3 only.
+ * #19: Fix for README mention of sample files not included in package.
+ * #23: Sample files for triggering borgmatic from a systemd timer.
+ * Support for backing up to multiple repositories.
+ * To free up space, now pruning backups prior to creating a new backup.
+ * Enabled test coverage output during tox runs.
+ * Added logo.
+
+1.0.3
+
+ * #22: Fix for verbosity flag not actually causing verbose output.
+
+1.0.2
+
+ * #21: Fix for traceback when remote_path option is missing.
+
+1.0.1
+
+ * #20: Support for Borg's --remote-path option to use an alternate Borg
+   executable. See sample/config.
+
 1.0.0
 
  * Attic is no longer supported, as there hasn't been any recent development on
@@ -17,13 +165,13 @@
 
 0.1.7
 
- * #11: Fixed parsing of punctuation in configuration file.
+ * #12: Fixed parsing of punctuation in configuration file.
  * Better error message when configuration file is missing.
 
 0.1.6
 
- * #9: New configuration option for the encryption passphrase.
- * #10: Support for Borg's new archive compression feature.
+ * #10: New configuration option for the encryption passphrase.
+ * #11: Support for Borg's new archive compression feature.
 
 0.1.5
 
@@ -35,7 +183,7 @@
 
 0.1.3
 
- * #1: Add support for "borg check --last N" to Borg backend.
+ * #2: Add support for "borg check --last N" to Borg backend.
 
 0.1.2
 

README.md

@@ -1,4 +1,4 @@
-title: Borgmatic
+<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" width="150px" style="float: right; padding-left: 1em;">
 
 ## Overview
 
@@ -11,62 +11,209 @@ all on the command-line, and handles common errors.
 
 Here's an example config file:
 
-```INI
-[location]
-# Space-separated list of source directories to backup.
-# Globs are expanded.
-source_directories: /home /etc /var/log/syslog*
+```yaml
+location:
+    # List of source directories to backup. Globs are expanded.
+    source_directories:
+        - /home
+        - /etc
+        - /var/log/syslog*
 
-# Path to local or remote backup repository.
-repository: user@backupserver:sourcehostname.borg
+    # Paths to local or remote repositories.
+    repositories:
+        - user@backupserver:sourcehostname.borg
 
-[retention]
-# Retention policy for how many backups to keep in each category.
-keep_daily: 7
-keep_weekly: 4
-keep_monthly: 6
+    # Any paths matching these patterns are excluded from backups.
+    exclude_patterns:
+        - /home/*/.cache
 
-[consistency]
-# Consistency checks to run, or "disabled" to prevent checks.
-checks: repository archives
+retention:
+    # Retention policy for how many backups to keep in each category.
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
+
+consistency:
+    # List of consistency checks to run: "repository", "archives", or both.
+    checks:
+        - repository
+        - archives
 ```
 
-Additionally, exclude patterns can be specified in a separate excludes config
-file, one pattern per line.
-
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://torsion.org/hg/borgmatic). It's also mirrored on
-[GitHub](https://github.com/witten/borgmatic) and
-[BitBucket](https://bitbucket.org/dhelfman/borgmatic) for convenience.
+available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
+on [GitHub](https://github.com/witten/borgmatic) for convenience.
+
+<a href="https://asciinema.org/a/164143" target="_blank"><img src="https://asciinema.org/a/164143.png" width="100%" /></a>
 
 
-## Setup
+## Installation
 
 To get up and running, follow the [Borg Quick
 Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host. Note that if you plan to run
-borgmatic on a schedule with cron, and you encrypt your Borg repository with
-a passphrase instead of a key file, you'll need to set the borgmatic
-`encryption_passphrase` configuration variable. See the repository encryption
-section of the Quick Start for more info.
+a repository on a local or remote host. Note that if you plan to run borgmatic
+on a schedule with cron, and you encrypt your Borg repository with a
+passphrase instead of a key file, you'll either need to set the borgmatic
+`encryption_passphrase` configuration variable or set the `BORG_PASSPHRASE`
+environment variable. See the [repository encryption
+section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
+of the Quick Start for more info.
+
+Alternatively, the passphrase can be specified programatically by setting
+either the borgmatic `encryption_passcommand` configuration variable or the
+`BORG_PASSCOMMAND` environment variable. See the [Borg Security
+FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
+for more info.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.
 
 To install borgmatic, run the following command to download and install it:
 
-    sudo pip install --upgrade borgmatic
+```bash
+sudo pip3 install --upgrade borgmatic
+```
 
-Then, copy the following configuration files:
+Note that your pip binary may have a different name than "pip3". Make sure
+you're using Python 3, as borgmatic does not support Python 2.
 
-    sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic
-    sudo mkdir /etc/borgmatic/
-    sudo cp sample/config sample/excludes /etc/borgmatic/
+### Other ways to install
 
-Lastly, modify the /etc files with your desired configuration.
+ * [A borgmatic Docker image](https://hub.docker.com/r/b3vis/borgmatic/) based
+   on Alpine Linux.
+ * [Another borgmatic Docker image](https://hub.docker.com/r/coaxial/borgmatic/) based
+   on Alpine Linux, updated automatically whenever the Alpine image updates.
+ * [A borgmatic package for
+   Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic).
+ * [A borgmatic package for Arch
+   Linux](https://aur.archlinux.org/packages/borgmatic/).
+ * [A borgmatic package for OpenBSD](http://ports.su/sysutils/borgmatic).
+<br><br>
+
+## Configuration
+
+After you install borgmatic, generate a sample configuration file:
+
+```bash
+sudo generate-borgmatic-config
+```
+
+If that command is not found, then it may be installed in a location that's
+not in your system `PATH`. Try looking in `/usr/local/bin/`.
+
+This generates a sample configuration file at /etc/borgmatic/config.yaml (by
+default). You should edit the file to suit your needs, as the values are just
+representative. All fields are optional except where indicated, so feel free
+to remove anything you don't need.
+
+You can also have a look at the [full configuration
+schema](https://projects.torsion.org/witten/borgmatic/src/master/borgmatic/config/schema.yaml)
+for the authoritative set of all configuration options. This is handy if
+borgmatic has added new options since you originally created your
+configuration file.
 
 
-## Upgrading from atticmatic
+### Multiple configuration files
+
+A more advanced usage is to create multiple separate configuration files and
+place each one in an /etc/borgmatic.d directory. For instance:
+
+```bash
+sudo mkdir /etc/borgmatic.d
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+```
+
+With this approach, you can have entirely different backup policies for
+different applications on your system. For instance, you may want one backup
+configuration for your database data directory, and a different configuration
+for your user home directories.
+
+When you set up multiple configuration files like this, borgmatic will run
+each one in turn from a single borgmatic invocation. This includes, by
+default, the traditional /etc/borgmatic/config.yaml as well.
+
+And if you need even more customizability, you can specify alternate
+configuration paths on the command-line with borgmatic's `--config` option.
+See `borgmatic --help` for more information.
+
+
+### Hooks
+
+If you find yourself performing prepraration tasks before your backup runs, or
+cleanup work afterwards, borgmatic hooks may be of interest. They're simply
+shell commands that borgmatic executes for you at various points, and they're
+configured in the `hooks` section of your configuration file.
+
+For instance, you can specify `before_backup` hooks to dump a database to file
+before backing it up, and specify `after_backup` hooks to delete the temporary
+file afterwards.
+
+borgmatic hooks run once per configuration file. `before_backup` hooks run
+prior to backups of all repositories. `after_backup` hooks run afterwards, but
+not if an error occurs in a previous hook or in the backups themselves. And
+borgmatic runs `on_error` hooks if an error occurs.
+
+An important security note about hooks: borgmatic executes all hook commands
+with the user permissions of borgmatic itself. So to prevent potential shell
+injection or privilege escalation, do not forget to set secure permissions
+(chmod 0700) on borgmatic configuration files and scripts invoked by hooks.
+
+See the sample generated configuration file mentioned above for specifics
+about hook configuration syntax.
+
+
+## Upgrading
+
+In general, all you should need to do to upgrade borgmatic is run the
+following:
+
+```bash
+sudo pip3 install --upgrade borgmatic
+```
+
+However, see below about special cases.
+
+
+### Upgrading from borgmatic 1.0.x
+
+borgmatic changed its configuration file format in version 1.1.0 from
+INI-style to YAML. This better supports validation, and has a more natural way
+to express lists of values. To upgrade your existing configuration, first
+upgrade to the new version of borgmatic.
+
+As of version 1.1.0, borgmatic no longer supports Python 2. If you were
+already running borgmatic with Python 3, then you can simply upgrade borgmatic
+in-place:
+
+```bash
+sudo pip3 install --upgrade borgmatic
+```
+
+But if you were running borgmatic with Python 2, uninstall and reinstall instead:
+
+```bash
+sudo pip uninstall borgmatic
+sudo pip3 install borgmatic
+```
+
+The pip binary names for different versions of Python can differ, so the above
+commands may need some tweaking to work on your machine.
+
+
+Once borgmatic is upgraded, run:
+
+```bash
+sudo upgrade-borgmatic-config
+```
+
+That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
+(by default) using the values from both your existing configuration and
+excludes files. The new version of borgmatic will consume the YAML
+configuration file instead of the old one.
+
+
+### Upgrading from atticmatic
 
 You can ignore this section if you're not an atticmatic user (the former name
 of borgmatic).
@@ -80,46 +227,193 @@ your borgmatic configuration files.
 If you were already using Borg with atticmatic, then you can easily upgrade
 from atticmatic to borgmatic. Simply run the following commands:
 
-    sudo pip uninstall atticmatic
-    sudo pip install borgmatic
+```bash
+sudo pip3 uninstall atticmatic
+sudo pip3 install borgmatic
+```
 
 That's it! borgmatic will continue using your /etc/borgmatic configuration
 files.
 
+
 ## Usage
 
 You can run borgmatic and start a backup simply by invoking it without
 arguments:
 
-    borgmatic
+```bash
+borgmatic
+```
 
 This will also prune any old backups as per the configured retention policy,
 and check backups for consistency problems due to things like file damage.
 
+If you'd like to see the available command-line arguments, view the help:
+
+```bash
+borgmatic --help
+```
+
+Note that borgmatic prunes archives *before* creating an archive, so as to
+free up space for archiving. This means that when a borgmatic run finishes,
+there may still be prune-able archives. Not to worry, as they will get cleaned
+up at the start of the next run.
+
+### Verbosity
+
 By default, the backup will proceed silently except in the case of errors. But
 if you'd like to to get additional information about the progress of the
 backup as it proceeds, use the verbosity option:
 
-    borgmatic --verbosity 1
+```bash
+borgmatic --verbosity 1
+```
 
 Or, for even more progress spew:
 
-    borgmatic --verbosity 2
+```bash
+borgmatic --verbosity 2
+```
 
-If you'd like to see the available command-line arguments, view the help:
+### À la carte
 
-    borgmatic --help
+If you want to run borgmatic with only pruning, creating, or checking enabled,
+the following optional flags are available:
+
+```bash
+borgmatic --prune
+borgmatic --create
+borgmatic --check
+```
+
+You can run with only one of these flags provided, or you can mix and match
+any number of them. This supports use cases like running consistency checks
+from a different cron job with a different frequency, or running pruning with
+a different verbosity level.
 
 
-## Running tests
+## Autopilot
 
-First install tox, which is used for setting up testing environments:
+If you want to run borgmatic automatically, say once a day, the you can
+configure a job runner to invoke it periodically.
 
-    pip install tox
+### cron
 
-Then, to actually run tests, run:
+If you're using cron, download the [sample cron
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic).
+Then, from the directory where you downloaded it:
 
-    tox
+```bash
+sudo mv borgmatic /etc/cron.d/borgmatic
+sudo chmod +x /etc/cron.d/borgmatic
+```
+
+You can modify the cron file if you'd like to run borgmatic more or less frequently.
+
+### systemd
+
+If you're using systemd instead of cron to run jobs, download the [sample
+systemd service
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.service)
+and the [sample systemd timer
+file](https://projects.torsion.org/witten/borgmatic/src/master/sample/systemd/borgmatic.timer).
+Then, from the directory where you downloaded them:
+
+```bash
+sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
+sudo systemctl enable borgmatic.timer
+sudo systemctl start borgmatic.timer
+```
+
+Feel free to modify the timer file based on how frequently you'd like
+borgmatic to run.
+
+
+## Support and contributing
+
+### Issues
+
+You've got issues? Or an idea for a feature enhancement? We've got an [issue
+tracker](https://projects.torsion.org/witten/borgmatic/issues). In order to
+create a new issue or comment on an issue, you'll need to [login
+first](https://projects.torsion.org/user/login). Note that you can login with
+an existing GitHub account if you prefer.
+
+Other questions or comments? Contact <mailto:witten@torsion.org>.
+
+
+### Contributing
+
+If you'd like to contribute to borgmatic development, please feel free to
+submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)
+or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
+to discuss your idea. We also accept Pull Requests on GitHub, if that's more
+your thing. In general, contributions are very welcome. We don't bite! 
+
+
+### Code style
+
+Start with [PEP 8](https://www.python.org/dev/peps/pep-0008/). But then, apply
+the following deviations from it:
+
+ * For strings, prefer single quotes over double quotes.
+ * Limit all lines to a maximum of 100 characters.
+ * Use trailing commas within multiline values or argument lists.
+ * For multiline constructs, put opening and closing delimeters on lines
+   separate from their contents.
+ * Within multiline constructs, use standard four-space indentation. Don't align
+   indentation with an opening delimeter.
+
+
+### Development
+
+To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
+
+```
+git clone https://projects.torsion.org/witten/borgmatic.git
+```
+
+Or:
+
+```bash
+git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git
+```
+
+Then, install borgmatic
+"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)"
+so that you can easily run borgmatic commands while you're hacking on them to
+make sure your changes work.
+
+```bash
+cd borgmatic/
+pip3 install --editable --user .
+```
+
+Note that this will typically install the borgmatic commands into
+`~/.local/bin`, which may or may not be on your PATH. There are other ways to
+install borgmatic editable as well, for instance into the system Python
+install (so without `--user`, as root), or even into a
+[virtualenv](https://virtualenv.pypa.io/en/stable/). How or where you install
+borgmatic is up to you, but generally an editable install makes development
+and testing easier.
+
+
+### Running tests
+
+Assuming you've cloned the borgmatic source code as described above, and
+you're in the `borgmatic/` working copy, install tox, which is used for
+setting up testing environments:
+
+```bash
+sudo pip3 install tox
+```
+
+Finally, to actually run tests, run:
+
+```bash
+cd borgmatic
+tox
+```
 
 
 ## Troubleshooting
@@ -130,24 +424,29 @@ When running borgmatic on a large remote repository, you may receive errors
 like the following, particularly while "borg check" is validating backups for
 consistency:
 
+```text
     Write failed: Broken pipe
     borg: Error: Connection closed by remote host
+```
 
 This error can be caused by an ssh timeout, which you can rectify by adding
 the following to the ~/.ssh/config file on the client:
 
+```text
     Host *
         ServerAliveInterval 120
+```
 
 This should make the client keep the connection alive while validating
 backups.
 
 
-## Issues and feedback
+### libyaml compilation errors
 
-Got an issue or an idea for a feature enhancement? Check out the [borgmatic
-issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In
-order to create a new issue or comment on an issue, you'll need to [login
-first](https://tree.taiga.io/login).
-
-Other questions or comments? Contact <mailto:witten@torsion.org>.
+borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally
+use a C YAML library (libyaml) if present. But if it's not installed, then
+when installing or upgrading borgmatic, you may see errors about compiling the
+YAML library. If so, not to worry. borgmatic should install and function
+correctly even without the C YAML library. And borgmatic won't be any faster
+with the C library present, so you don't need to go out of your way to install
+it.

borgmatic/borg.py

@@ -1,183 +0,0 @@
-from datetime import datetime
-import os
-import re
-import platform
-import subprocess
-from glob import glob
-from itertools import chain
-
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-# Integration with Borg for actually handling backups.
-
-
-COMMAND = 'borg'
-
-
-def initialize(storage_config, command=COMMAND):
-    passphrase = storage_config.get('encryption_passphrase')
-
-    if passphrase:
-        os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
-
-
-def create_archive(
-    excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND,
-    one_file_system=None
-):
-    '''
-    Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
-    list of source directories, a local or remote repository path, and a command to run, create an
-    attic archive.
-    '''
-    sources = re.split('\s+', source_directories)
-    sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources))
-    exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
-    compression = storage_config.get('compression', None)
-    compression_flags = ('--compression', compression) if compression else ()
-    umask = storage_config.get('umask', None)
-    umask_flags = ('--umask', str(umask)) if umask else ()
-    one_file_system_flags = ('--one-file-system',) if one_file_system else ()
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--stats',),
-        VERBOSITY_LOTS: ('--verbose', '--stats'),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'create',
-        '{repo}::{hostname}-{timestamp}'.format(
-            repo=repository,
-            hostname=platform.node(),
-            timestamp=datetime.now().isoformat(),
-        ),
-    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
-        umask_flags + verbosity_flags
-
-    subprocess.check_call(full_command)
-
-
-def _make_prune_flags(retention_config):
-    '''
-    Given a retention config dict mapping from option name to value, tranform it into an iterable of
-    command-line name-value flag pairs.
-
-    For example, given a retention config of:
-
-        {'keep_weekly': 4, 'keep_monthly': 6}
-
-    This will be returned as an iterable of:
-
-        (
-            ('--keep-weekly', '4'),
-            ('--keep-monthly', '6'),
-        )
-    '''
-    return (
-        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
-        for option_name, value in retention_config.items()
-    )
-
-
-def prune_archives(verbosity, repository, retention_config, command=COMMAND):
-    '''
-    Given a verbosity flag, a local or remote repository path, a retention config dict, and a
-    command to run, prune attic archives according the the retention policy specified in that
-    configuration.
-    '''
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--stats',),
-        VERBOSITY_LOTS: ('--verbose', '--stats'),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'prune',
-        repository,
-    ) + tuple(
-        element
-        for pair in _make_prune_flags(retention_config)
-        for element in pair
-    ) + verbosity_flags
-
-    subprocess.check_call(full_command)
-
-
-DEFAULT_CHECKS = ('repository', 'archives')
-
-
-def _parse_checks(consistency_config):
-    '''
-    Given a consistency config with a space-separated "checks" option, transform it to a tuple of
-    named checks to run.
-
-    For example, given a retention config of:
-
-        {'checks': 'repository archives'}
-
-    This will be returned as:
-
-        ('repository', 'archives')
-
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
-    '''
-    checks = consistency_config.get('checks', '').strip()
-    if not checks:
-        return DEFAULT_CHECKS
-
-    return tuple(
-        check for check in consistency_config['checks'].split(' ')
-        if check.lower() not in ('disabled', '')
-    )
-
-
-def _make_check_flags(checks, check_last=None):
-    '''
-    Given a parsed sequence of checks, transform it into tuple of command-line flags.
-
-    For example, given parsed checks of:
-
-        ('repository',)
-
-    This will be returned as:
-
-        ('--repository-only',)
-
-    Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
-    Borg supports this flag.
-    '''
-    last_flag = ('--last', check_last) if check_last else ()
-    if checks == DEFAULT_CHECKS:
-        return last_flag
-
-    return tuple(
-        '--{}-only'.format(check) for check in checks
-    ) + last_flag
-
-
-def check_archives(verbosity, repository, consistency_config, command=COMMAND):
-    '''
-    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    command to run, check the contained attic archives for consistency.
-
-    If there are no consistency checks to run, skip running them.
-    '''
-    checks = _parse_checks(consistency_config)
-    check_last = consistency_config.get('check_last', None)
-    if not checks:
-        return
-
-    verbosity_flags = {
-        VERBOSITY_SOME: ('--verbose',),
-        VERBOSITY_LOTS: ('--verbose',),
-    }.get(verbosity, ())
-
-    full_command = (
-        command, 'check',
-        repository,
-    ) + _make_check_flags(checks, check_last) + verbosity_flags
-
-    # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
-    stdout = None if verbosity_flags else open(os.devnull, 'w')
-
-    subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)

borgmatic/borg/check.py

@@ -0,0 +1,112 @@
+import logging
+import os
+import subprocess
+
+from borgmatic.borg import extract
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+DEFAULT_CHECKS = ('repository', 'archives')
+DEFAULT_PREFIX = '{hostname}-'
+
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_checks(consistency_config):
+    '''
+    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
+
+    For example, given a retention config of:
+
+        {'checks': ['repository', 'archives']}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
+    "disabled", return an empty tuple, meaning that no checks should be run.
+    '''
+    checks = consistency_config.get('checks', [])
+    if checks == ['disabled']:
+        return ()
+
+    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
+
+
+def _make_check_flags(checks, check_last=None, prefix=None):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+
+        ('--repository-only',)
+
+    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    flags because Borg does both checks by default.
+
+    Additionally, if a check_last value is given and "archives" is in checks, then include a
+    "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
+    "--prefix" flag.
+    '''
+    if 'archives' in checks:
+        last_flags = ('--last', str(check_last)) if check_last else ()
+        prefix_flags = ('--prefix', prefix) if prefix else ('--prefix', DEFAULT_PREFIX)
+    else:
+        last_flags = ()
+        prefix_flags = ()
+        if check_last:
+            logger.warn('Ignoring check_last option, as "archives" is not in consistency checks.')
+        if prefix:
+            logger.warn('Ignoring consistency prefix option, as "archives" is not in consistency checks.')
+        
+    if set(DEFAULT_CHECKS).issubset(set(checks)):
+        return last_flags + prefix_flags
+
+    return tuple(
+        '--{}-only'.format(check) for check in checks
+        if check in DEFAULT_CHECKS
+    ) + last_flags + prefix_flags
+
+
+def check_archives(verbosity, repository, storage_config, consistency_config, local_path='borg',
+                   remote_path=None):
+    '''
+    Given a verbosity flag, a local or remote repository path, a storage config dict, a consistency
+    config dict, and a local/remote commands to run, check the contained Borg archives for
+    consistency.
+
+    If there are no consistency checks to run, skip running them.
+    '''
+    checks = _parse_checks(consistency_config)
+    check_last = consistency_config.get('check_last', None)
+    lock_wait = None
+
+    if set(checks).intersection(set(DEFAULT_CHECKS)):
+        remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+        lock_wait = storage_config.get('lock_wait', None)
+        lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
+        verbosity_flags = {
+            VERBOSITY_SOME: ('--info',),
+            VERBOSITY_LOTS: ('--debug',),
+        }.get(verbosity, ())
+        prefix = consistency_config.get('prefix')
+
+        full_command = (
+            local_path, 'check',
+            repository,
+        ) + _make_check_flags(checks, check_last, prefix) + remote_path_flags + lock_wait_flags + verbosity_flags
+
+        # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
+        stdout = None if verbosity_flags else open(os.devnull, 'w')
+
+        logger.debug(' '.join(full_command))
+        subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
+
+    if 'extract' in checks:
+        extract.extract_last_archive_dry_run(verbosity, repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -0,0 +1,159 @@
+import glob
+import itertools
+import logging
+import os
+import subprocess
+import tempfile
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+logger = logging.getLogger(__name__)
+
+
+def initialize_environment(storage_config):
+    passcommand = storage_config.get('encryption_passcommand')
+    if passcommand:
+        os.environ['BORG_PASSCOMMAND'] = passcommand
+
+    passphrase = storage_config.get('encryption_passphrase')
+    if passphrase:
+        os.environ['BORG_PASSPHRASE'] = passphrase
+
+    ssh_command = storage_config.get('ssh_command')
+    if ssh_command:
+        os.environ['BORG_RSH'] = ssh_command
+
+
+def _expand_directory(directory):
+    '''
+    Given a directory path, expand any tilde (representing a user's home directory) and any globs
+    therein. Return a list of one or more resulting paths.
+    '''
+    expanded_directory = os.path.expanduser(directory)
+
+    return glob.glob(expanded_directory) or [expanded_directory]
+
+
+def _expand_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(
+            _expand_directory(directory)
+            for directory in directories
+        )
+    )
+
+
+def _write_pattern_file(patterns=None):
+    '''
+    Given a sequence of patterns, write them to a named temporary file and return it. Return None
+    if no patterns are provided.
+    '''
+    if not patterns:
+        return None
+
+    pattern_file = tempfile.NamedTemporaryFile('w')
+    pattern_file.write('\n'.join(patterns))
+    pattern_file.flush()
+
+    return pattern_file
+
+
+def _make_pattern_flags(location_config, pattern_filename=None):
+    '''
+    Given a location config dict with a potential pattern_from option, and a filename containing any
+    additional patterns, return the corresponding Borg flags for those files as a tuple.
+    '''
+    pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
+        (pattern_filename,) if pattern_filename else ()
+    )
+
+    return tuple(
+        itertools.chain.from_iterable(
+            ('--patterns-from', pattern_filename)
+            for pattern_filename in pattern_filenames
+        )
+    )
+
+
+def _make_exclude_flags(location_config, exclude_filename=None):
+    '''
+    Given a location config dict with various exclude options, and a filename containing any exclude
+    patterns, return the corresponding Borg flags as a tuple.
+    '''
+    exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
+        (exclude_filename,) if exclude_filename else ()
+    )
+    exclude_from_flags = tuple(
+        itertools.chain.from_iterable(
+            ('--exclude-from', exclude_filename)
+            for exclude_filename in exclude_filenames
+        )
+    )
+    caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
+    if_present = location_config.get('exclude_if_present')
+    if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+
+    return exclude_from_flags + caches_flag + if_present_flags
+
+
+def create_archive(
+    verbosity, dry_run, repository, location_config, storage_config, local_path='borg', remote_path=None,
+):
+    '''
+    Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
+    storage config dict, create a Borg archive.
+    '''
+    sources = _expand_directories(location_config['source_directories'])
+
+    pattern_file = _write_pattern_file(location_config.get('patterns'))
+    exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
+    compression = storage_config.get('compression', None)
+    remote_rate_limit = storage_config.get('remote_rate_limit', None)
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+    files_cache = location_config.get('files_cache')
+    default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+    archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
+
+    full_command = (
+        (
+            local_path, 'create',
+            '{repository}::{archive_name_format}'.format(
+                repository=repository,
+                archive_name_format=archive_name_format,
+            ),
+        )
+        + sources
+        + _make_pattern_flags(
+            location_config,
+            pattern_file.name if pattern_file else None,
+        )
+        + _make_exclude_flags(
+            location_config,
+            exclude_file.name if exclude_file else None,
+        )
+        + (('--compression', compression) if compression else ())
+        + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
+        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
+        + (('--files-cache', files_cache) if files_cache else ())
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + {
+            VERBOSITY_SOME: ('--info',) if dry_run else ('--info', '--stats',),
+            VERBOSITY_LOTS: ('--debug', '--list',) if dry_run else ('--debug', '--list', '--stats',),
+        }.get(verbosity, ())
+        + (('--dry-run',) if dry_run else ())
+    )
+
+    logger.debug(' '.join(full_command))
+    subprocess.check_call(full_command)

borgmatic/borg/extract.py

@@ -0,0 +1,46 @@
+import logging
+import sys
+import subprocess
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+logger = logging.getLogger(__name__)
+
+
+def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_path='borg', remote_path=None):
+    '''
+    Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
+    the dry-run.
+    '''
+    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
+    verbosity_flags = {
+        VERBOSITY_SOME: ('--info',),
+        VERBOSITY_LOTS: ('--debug',),
+    }.get(verbosity, ())
+
+    full_list_command = (
+        local_path, 'list',
+        '--short',
+        repository,
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags
+
+    list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
+
+    last_archive_name = list_output.strip().split('\n')[-1]
+    if not last_archive_name:
+        return
+
+    list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
+    full_extract_command = (
+        local_path, 'extract',
+        '--dry-run',
+        '{repository}::{last_archive_name}'.format(
+            repository=repository,
+            last_archive_name=last_archive_name,
+        ),
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag
+
+    logger.debug(' '.join(full_extract_command))
+    subprocess.check_call(full_extract_command)

borgmatic/borg/info.py

@@ -0,0 +1,33 @@
+import logging
+import subprocess
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+logger = logging.getLogger(__name__)
+
+
+def display_archives_info(
+    verbosity, repository, storage_config, local_path='borg', remote_path=None, json=False
+):
+    '''
+    Given a verbosity flag, a local or remote repository path, and a storage config dict,
+    display summary information for Borg archives in the repository.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'info', repository)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--json',) if json else ())
+        + {
+            VERBOSITY_SOME: ('--info',),
+            VERBOSITY_LOTS: ('--debug',),
+        }.get(verbosity, ())
+    )
+
+    logger.debug(' '.join(full_command))
+
+    output = subprocess.check_output(full_command)
+    return output.decode() if output is not None else None

borgmatic/borg/list.py

@@ -0,0 +1,33 @@
+import logging
+import subprocess
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+logger = logging.getLogger(__name__)
+
+
+def list_archives(
+    verbosity, repository, storage_config, local_path='borg', remote_path=None, json=False
+):
+    '''
+    Given a verbosity flag, a local or remote repository path, and a storage config dict,
+    list Borg archives in the repository.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'list', repository)
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--json',) if json else ())
+        + {
+            VERBOSITY_SOME: ('--info',),
+            VERBOSITY_LOTS: ('--debug',),
+        }.get(verbosity, ())
+    )
+
+    logger.debug(' '.join(full_command))
+
+    output = subprocess.check_output(full_command)
+    return output.decode() if output is not None else None

borgmatic/borg/prune.py

@@ -0,0 +1,65 @@
+import logging
+import subprocess
+
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+logger = logging.getLogger(__name__)
+
+
+def _make_prune_flags(retention_config):
+    '''
+    Given a retention config dict mapping from option name to value, tranform it into an iterable of
+    command-line name-value flag pairs.
+
+    For example, given a retention config of:
+
+        {'keep_weekly': 4, 'keep_monthly': 6}
+
+    This will be returned as an iterable of:
+
+        (
+            ('--keep-weekly', '4'),
+            ('--keep-monthly', '6'),
+        )
+    '''
+    if not retention_config.get('prefix'):
+        retention_config['prefix'] = '{hostname}-'
+
+    return (
+        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
+        for option_name, value in retention_config.items()
+    )
+
+
+def prune_archives(verbosity, dry_run, repository, storage_config, retention_config,
+                   local_path='borg', remote_path=None):
+    '''
+    Given verbosity/dry-run flags, a local or remote repository path, a storage config dict, and a
+    retention config dict, prune Borg archives according to the retention policy specified in that
+    configuration.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (
+            local_path, 'prune',
+            repository,
+        ) + tuple(
+            element
+            for pair in _make_prune_flags(retention_config)
+            for element in pair
+        )
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + {
+            VERBOSITY_SOME: ('--info', '--stats',),
+            VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
+        }.get(verbosity, ())
+        + (('--dry-run',) if dry_run else ())
+    )
+
+    logger.debug(' '.join(full_command))
+    subprocess.check_call(full_command)

borgmatic/command.py

@@ -1,57 +0,0 @@
-from __future__ import print_function
-from argparse import ArgumentParser
-import os
-from subprocess import CalledProcessError
-import sys
-
-from borgmatic import borg
-from borgmatic.config import parse_configuration, CONFIG_FORMAT
-
-
-DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'
-DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
-
-
-def parse_arguments(*arguments):
-    '''
-    Given the name of the command with which this script was invoked and command-line arguments,
-    parse the arguments and return them as an ArgumentParser instance. Use the command name to
-    determine the default configuration and excludes paths.
-    '''
-    parser = ArgumentParser()
-    parser.add_argument(
-        '-c', '--config',
-        dest='config_filename',
-        default=DEFAULT_CONFIG_FILENAME,
-        help='Configuration filename',
-    )
-    parser.add_argument(
-        '--excludes',
-        dest='excludes_filename',
-        default=DEFAULT_EXCLUDES_FILENAME if os.path.exists(DEFAULT_EXCLUDES_FILENAME) else None,
-        help='Excludes filename',
-    )
-    parser.add_argument(
-        '-v', '--verbosity',
-        type=int,
-        help='Display verbose progress (1 for some, 2 for lots)',
-    )
-
-    return parser.parse_args(arguments)
-
-
-def main():
-    try:
-        args = parse_arguments(*sys.argv[1:])
-        config = parse_configuration(args.config_filename, CONFIG_FORMAT)
-        repository = config.location['repository']
-
-        borg.initialize(config.storage)
-        borg.create_archive(
-            args.excludes_filename, args.verbosity, config.storage, **config.location
-        )
-        borg.prune_archives(args.verbosity, repository, config.retention)
-        borg.check_archives(args.verbosity, repository, config.consistency)
-    except (ValueError, IOError, CalledProcessError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)

borgmatic/commands/borgmatic.py

@@ -0,0 +1,244 @@
+from argparse import ArgumentParser
+import json
+import logging
+import os
+from subprocess import CalledProcessError
+import sys
+
+from borgmatic.borg import check as borg_check, create as borg_create, prune as borg_prune, \
+     list as borg_list, info as borg_info
+from borgmatic.commands import hook
+from borgmatic.config import collect, convert, validate
+from borgmatic.signals import configure_signals
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS, verbosity_to_log_level
+
+
+logger = logging.getLogger(__name__)
+
+
+LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    config_paths = collect.get_default_config_paths()
+
+    parser = ArgumentParser(
+        description=
+            '''
+            A simple wrapper script for the Borg backup software that creates and prunes backups.
+            If none of the --prune, --create, or --check options are given, then borgmatic defaults
+            to all three: prune, create, and check archives.
+            '''
+    )
+    parser.add_argument(
+        '-c', '--config',
+        nargs='+',
+        dest='config_paths',
+        default=config_paths,
+        help='Configuration filenames or directories, defaults to: {}'.format(' '.join(config_paths)),
+    )
+    parser.add_argument(
+        '--excludes',
+        dest='excludes_filename',
+        help='Deprecated in favor of exclude_patterns within configuration',
+    )
+    parser.add_argument(
+        '-p', '--prune',
+        dest='prune',
+        action='store_true',
+        help='Prune archives according to the retention policy',
+    )
+    parser.add_argument(
+        '-C', '--create',
+        dest='create',
+        action='store_true',
+        help='Create archives (actually perform backups)',
+    )
+    parser.add_argument(
+        '-k', '--check',
+        dest='check',
+        action='store_true',
+        help='Check archives for consistency',
+    )
+    parser.add_argument(
+        '-l', '--list',
+        dest='list',
+        action='store_true',
+        help='List archives',
+    )
+    parser.add_argument(
+        '-i', '--info',
+        dest='info',
+        action='store_true',
+        help='Display summary information on archives',
+    )
+    parser.add_argument(
+        '--json',
+        dest='json',
+        default=False,
+        action='store_true',
+        help='Output results from the --list option as json',
+    )
+    parser.add_argument(
+        '-n', '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write to any repositories',
+    )
+    parser.add_argument(
+        '-v', '--verbosity',
+        type=int,
+        help='Display verbose progress (1 for some, 2 for lots)',
+    )
+
+    args = parser.parse_args(arguments)
+
+    if args.json and not (args.list or args.info):
+        raise ValueError('The --json option can only be used with the --list or --info options')
+
+    if args.json and args.list and args.info:
+        raise ValueError(
+            'With the --json option, options --list and --info cannot be used together'
+        )
+
+    # If any of the action flags are explicitly requested, leave them as-is. Otherwise, assume
+    # defaults: Mutate the given arguments to enable the default actions.
+    if args.prune or args.create or args.check or args.list or args.info:
+        return args
+
+    args.prune = True
+    args.create = True
+    args.check = True
+    return args
+
+
+def run_configuration(config_filename, args):  # pragma: no cover
+    '''
+    Parse a single configuration file, and execute its defined pruning, backups, and/or consistency
+    checks.
+    '''
+    logger.info('{}: Parsing configuration file'.format(config_filename))
+    config = validate.parse_configuration(config_filename, validate.schema_filename())
+    (location, storage, retention, consistency, hooks) = (
+        config.get(section_name, {})
+        for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks')
+    )
+
+    try:
+        local_path = location.get('local_path', 'borg')
+        remote_path = location.get('remote_path')
+        borg_create.initialize_environment(storage)
+
+        if args.create:
+            hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
+
+        _run_commands(args, consistency, local_path, location, remote_path, retention, storage)
+
+        if args.create:
+            hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
+    except (OSError, CalledProcessError):
+        hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error')
+        raise
+
+
+def _run_commands(args, consistency, local_path, location, remote_path, retention, storage):
+    json_results = []
+    for unexpanded_repository in location['repositories']:
+        _run_commands_on_repository(
+            args, consistency, json_results, local_path, location, remote_path, retention, storage,
+            unexpanded_repository,
+        )
+    if args.json:
+        sys.stdout.write(json.dumps(json_results))
+
+
+def _run_commands_on_repository(
+    args, consistency, json_results, local_path, location, remote_path,
+    retention, storage, unexpanded_repository,
+):  # pragma: no cover
+    repository = os.path.expanduser(unexpanded_repository)
+    dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
+    if args.prune:
+        logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
+        borg_prune.prune_archives(
+            args.verbosity,
+            args.dry_run,
+            repository,
+            storage,
+            retention,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+    if args.create:
+        logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        borg_create.create_archive(
+            args.verbosity,
+            args.dry_run,
+            repository,
+            location,
+            storage,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+    if args.check:
+        logger.info('{}: Running consistency checks'.format(repository))
+        borg_check.check_archives(
+            args.verbosity,
+            repository,
+            storage,
+            consistency,
+            local_path=local_path,
+            remote_path=remote_path,
+        )
+    if args.list:
+        logger.info('{}: Listing archives'.format(repository))
+        output = borg_list.list_archives(
+            args.verbosity,
+            repository,
+            storage,
+            local_path=local_path,
+            remote_path=remote_path,
+            json=args.json,
+        )
+        if args.json:
+            json_results.append(json.loads(output))
+        else:
+            sys.stdout.write(output)
+    if args.info:
+        logger.info('{}: Displaying summary info for archives'.format(repository))
+        output = borg_info.display_archives_info(
+            args.verbosity,
+            repository,
+            storage,
+            local_path=local_path,
+            remote_path=remote_path,
+            json=args.json,
+        )
+        if args.json:
+            json_results.append(json.loads(output))
+        else:
+            sys.stdout.write(output)
+
+
+def main():  # pragma: no cover
+    try:
+        configure_signals()
+        args = parse_arguments(*sys.argv[1:])
+        logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s')
+
+        config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
+        logger.debug('Ensuring legacy configuration is upgraded')
+        convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
+
+        if len(config_filenames) == 0:
+            raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths)))
+
+        for config_filename in config_filenames:
+            run_configuration(config_filename, args)
+    except (ValueError, OSError, CalledProcessError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)

borgmatic/commands/convert_config.py

@@ -0,0 +1,97 @@
+from argparse import ArgumentParser
+import os
+from subprocess import CalledProcessError
+import sys
+import textwrap
+
+from ruamel import yaml
+
+from borgmatic.config import convert, generate, legacy, validate
+
+
+DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
+DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
+DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    parser = ArgumentParser(
+        description='''
+            Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
+            configuration file. Note that this replaces any comments from the source files.
+        '''
+    )
+    parser.add_argument(
+        '-s', '--source-config',
+        dest='source_config_filename',
+        default=DEFAULT_SOURCE_CONFIG_FILENAME,
+        help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
+    )
+    parser.add_argument(
+        '-e', '--source-excludes',
+        dest='source_excludes_filename',
+        default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None,
+        help='Excludes filename',
+    )
+    parser.add_argument(
+        '-d', '--destination-config',
+        dest='destination_config_filename',
+        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
+        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+    )
+
+    return parser.parse_args(arguments)
+
+
+TEXT_WRAP_CHARACTERS = 80
+
+
+def display_result(args):  # pragma: no cover
+    result_lines = textwrap.wrap(
+        'Your borgmatic configuration has been upgraded. Please review the result in {}.'.format(
+            args.destination_config_filename
+        ),
+        TEXT_WRAP_CHARACTERS,
+    )
+    
+    delete_lines = textwrap.wrap(
+        'Once you are satisfied, you can safely delete {}{}.'.format(
+            args.source_config_filename,
+            ' and {}'.format(args.source_excludes_filename) if args.source_excludes_filename else '',
+        ),
+        TEXT_WRAP_CHARACTERS,
+    )
+
+    print('\n'.join(result_lines))
+    print()
+    print('\n'.join(delete_lines))
+
+
+def main():  # pragma: no cover
+    try:
+        args = parse_arguments(*sys.argv[1:])
+        schema = yaml.round_trip_load(open(validate.schema_filename()).read())
+        source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
+        source_config_file_mode = os.stat(args.source_config_filename).st_mode
+        source_excludes = (
+            open(args.source_excludes_filename).read().splitlines()
+            if args.source_excludes_filename
+            else []
+        )
+
+        destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
+
+        generate.write_configuration(
+            args.destination_config_filename,
+            destination_config,
+            mode=source_config_file_mode,
+        )
+
+        display_result(args)
+    except (ValueError, OSError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)

borgmatic/commands/generate_config.py

@@ -0,0 +1,40 @@
+from argparse import ArgumentParser
+import os
+from subprocess import CalledProcessError
+import sys
+
+from borgmatic.config import convert, generate, validate
+
+
+DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+
+
+def parse_arguments(*arguments):
+    '''
+    Given command-line arguments with which this script was invoked, parse the arguments and return
+    them as an ArgumentParser instance.
+    '''
+    parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
+    parser.add_argument(
+        '-d', '--destination',
+        dest='destination_filename',
+        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
+        help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
+    )
+
+    return parser.parse_args(arguments)
+
+
+def main():  # pragma: no cover
+    try:
+        args = parse_arguments(*sys.argv[1:])
+
+        generate.generate_sample_configuration(args.destination_filename, validate.schema_filename())
+
+        print('Generated a sample configuration file at {}.'.format(args.destination_filename))
+        print()
+        print('Please edit the file to suit your needs. The values are just representative.')
+        print('All fields are optional except where indicated.')
+    except (ValueError, OSError) as error:
+        print(error, file=sys.stderr)
+        sys.exit(1)

borgmatic/commands/hook.py

@@ -0,0 +1,20 @@
+import logging
+import subprocess
+
+
+logger = logging.getLogger(__name__)
+
+
+def execute_hook(commands, config_filename, description):
+    if not commands:
+        logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))
+        return
+
+    if len(commands) == 1:
+        logger.info('{}: Running command for {} hook'.format(config_filename, description))
+    else:
+        logger.info('{}: Running {} commands for {} hook'.format(config_filename, len(commands), description))
+
+    for command in commands:
+        logger.debug('{}: Hook command: {}'.format(config_filename, command))
+        subprocess.check_call(command, shell=True)

borgmatic/config/collect.py

@@ -0,0 +1,47 @@
+import os
+
+
+def get_default_config_paths():
+    '''
+    Based on the value of the XDG_CONFIG_HOME and HOME environment variables, return a list of
+    default configuration paths. This includes both system-wide configuration and configuration in
+    the current user's home directory.
+    '''
+    user_config_directory = (
+        os.getenv('XDG_CONFIG_HOME') or os.path.expandvars(os.path.join('$HOME', '.config'))
+    )
+
+    return [
+        '/etc/borgmatic/config.yaml',
+        '/etc/borgmatic.d',
+        '%s/borgmatic/config.yaml' % user_config_directory,
+    ]
+
+
+def collect_config_filenames(config_paths):
+    '''
+    Given a sequence of config paths, both filenames and directories, resolve that to just an
+    iterable of files. Accomplish this by listing any given directories looking for contained config
+    files (ending with the ".yaml" extension). This is non-recursive, so any directories within the
+    given directories are ignored.
+
+    Return paths even if they don't exist on disk, so the user can find out about missing
+    configuration paths. However, skip a default config path if it's missing, so the user doesn't
+    have to create a default config path unless they need it.
+    '''
+    real_default_config_paths = set(map(os.path.realpath, get_default_config_paths()))
+
+    for path in config_paths:
+        exists = os.path.exists(path)
+
+        if os.path.realpath(path) in real_default_config_paths and not exists:
+            continue
+
+        if not os.path.isdir(path) or not exists:
+            yield path
+            continue
+
+        for filename in os.listdir(path):
+            full_filename = os.path.join(path, filename)
+            if full_filename.endswith('.yaml') and not os.path.isdir(full_filename):
+                yield full_filename

borgmatic/config/convert.py

@@ -0,0 +1,111 @@
+import os
+
+from ruamel import yaml
+
+from borgmatic.config import generate
+
+
+def _convert_section(source_section_config, section_schema):
+    '''
+    Given a legacy Parsed_config instance for a single section, convert it to its corresponding
+    yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
+
+    Where integer types exist in the given section schema, convert their values to integers.
+    '''
+    destination_section_config = yaml.comments.CommentedMap([
+        (
+            option_name,
+            int(option_value)
+                if section_schema['map'].get(option_name, {}).get('type') == 'int' else option_value
+        )
+        for option_name, option_value in source_section_config.items()
+    ])
+
+    return destination_section_config
+
+
+def convert_legacy_parsed_config(source_config, source_excludes, schema):
+    '''
+    Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
+    patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
+    preparation for serialization to a single YAML config file.
+
+    Additionally, use the given schema as a source of helpful comments to include within the
+    returned CommentedMap.
+    '''
+    destination_config = yaml.comments.CommentedMap([
+        (section_name, _convert_section(section_config, schema['map'][section_name]))
+        for section_name, section_config in source_config._asdict().items()
+    ])
+
+    # Split space-seperated values into actual lists, make "repository" into a list, and merge in
+    # excludes.
+    location = destination_config['location']
+    location['source_directories'] = source_config.location['source_directories'].split(' ')
+    location['repositories'] = [location.pop('repository')]
+    location['exclude_patterns'] = source_excludes
+
+    if source_config.consistency.get('checks'):
+        destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
+
+    # Add comments to each section, and then add comments to the fields in each section.
+    generate.add_comments_to_configuration(destination_config, schema)
+
+    for section_name, section_config in destination_config.items():
+        generate.add_comments_to_configuration(
+            section_config,
+            schema['map'][section_name],
+            indent=generate.INDENT,
+        )
+
+    return destination_config
+
+
+class LegacyConfigurationNotUpgraded(FileNotFoundError):
+    def __init__(self):
+        super(LegacyConfigurationNotUpgraded, self).__init__(
+            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
+to YAML. This better supports validation, and has a more natural way to express
+lists of values. To upgrade your existing configuration, run:
+
+    sudo upgrade-borgmatic-config
+
+That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
+(by default) using the values from both your existing configuration and excludes
+files. The new version of borgmatic will consume the YAML configuration file
+instead of the old one.'''
+        )
+
+
+def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
+    '''
+    If legacy source configuration exists but no destination upgraded configs do, raise
+    LegacyConfigurationNotUpgraded.
+
+    The idea is that we want to alert the user about upgrading their config if they haven't already.
+    '''
+    destination_config_exists = any(
+        os.path.exists(filename)
+        for filename in destination_config_filenames
+    )
+
+    if os.path.exists(source_config_filename) and not destination_config_exists:
+        raise LegacyConfigurationNotUpgraded()
+
+
+class LegacyExcludesFilenamePresent(FileNotFoundError):
+    def __init__(self):
+        super(LegacyExcludesFilenamePresent, self).__init__(
+            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
+to YAML. This better supports validation, and has a more natural way to express
+lists of values. The new configuration file incorporates excludes, so you no
+longer need to provide an excludes filename on the command-line with an
+"--excludes" argument.
+
+Please remove the "--excludes" argument and run borgmatic again.'''
+        )
+
+
+def guard_excludes_filename_omitted(excludes_filename):
+    if excludes_filename != None:
+        raise LegacyExcludesFilenamePresent()

borgmatic/config/generate.py

@@ -0,0 +1,93 @@
+from collections import OrderedDict
+import os
+
+from ruamel import yaml
+
+
+INDENT = 4
+
+
+def _insert_newline_before_comment(config, field_name):
+    '''
+    Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
+    field and its comments.
+    '''
+    config.ca.items[field_name][1].insert(
+        0,
+        yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
+    )
+
+
+def _schema_to_sample_configuration(schema, level=0):
+    '''
+    Given a loaded configuration schema, generate and return sample config for it. Include comments
+    for each section based on the schema "desc" description.
+    '''
+    example = schema.get('example')
+    if example is not None:
+        return example
+
+    config = yaml.comments.CommentedMap([
+        (
+            section_name,
+            _schema_to_sample_configuration(section_schema, level + 1),
+        )
+        for section_name, section_schema in schema['map'].items()
+    ])
+
+    add_comments_to_configuration(config, schema, indent=(level * INDENT))
+
+    return config
+
+
+def write_configuration(config_filename, config, mode=0o600):
+    '''
+    Given a target config filename and a config data structure of nested OrderedDicts, write out the
+    config to file as YAML. Create any containing directories as needed.
+    '''
+    if os.path.exists(config_filename):
+        raise FileExistsError('{} already exists. Aborting.'.format(config_filename))
+
+    try:
+        os.makedirs(os.path.dirname(config_filename), mode=0o700)
+    except (FileExistsError, FileNotFoundError):
+        pass
+
+    with open(config_filename, 'w') as config_file:
+        config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
+
+    os.chmod(config_filename, mode)
+
+
+def add_comments_to_configuration(config, schema, indent=0):
+    '''
+    Using descriptions from a schema as a source, add those descriptions as comments to the given
+    config before each field. This function only adds comments for the top-most config map level.
+    Indent the comment the given number of characters.
+    '''
+    for index, field_name in enumerate(config.keys()):
+        field_schema = schema['map'].get(field_name, {})
+        description = field_schema.get('desc')
+
+        # No description to use? Skip it.
+        if not field_schema or not description:
+            continue
+
+        config.yaml_set_comment_before_after_key(
+            key=field_name,
+            before=description,
+            indent=indent,
+        )
+        if index > 0:
+            _insert_newline_before_comment(config, field_name)
+
+
+def generate_sample_configuration(config_filename, schema_filename):
+    '''
+    Given a target config filename and the path to a schema filename in pykwalify YAML schema
+    format, write out a sample configuration file based on that schema.
+    '''
+    schema = yaml.round_trip_load(open(schema_filename))
+    config = _schema_to_sample_configuration(schema)
+
+    write_configuration(config_filename, config)

borgmatic/config/legacy.py

@@ -1,11 +1,5 @@
 from collections import OrderedDict, namedtuple
-
-try:
-    # Python 2
-    from ConfigParser import RawConfigParser
-except ImportError:
-    # Python 3
-    from configparser import RawConfigParser
+from configparser import RawConfigParser
 
 
 Section_format = namedtuple('Section_format', ('name', 'options'))
@@ -26,6 +20,7 @@ CONFIG_FORMAT = (
         (
             option('source_directories'),
             option('one_file_system', value_type=bool, required=False),
+            option('remote_path', required=False),
             option('repository'),
         ),
     ),

borgmatic/config/schema.yaml

@@ -0,0 +1,257 @@
+name: Borgmatic configuration file schema
+version: 1
+map:
+    location:
+        desc: |
+            Where to look for files to backup, and where to store those backups. See
+            https://borgbackup.readthedocs.io/en/stable/quickstart.html and
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
+        required: true
+        map:
+            source_directories:
+                required: true
+                seq:
+                    - type: scalar
+                desc: |
+                    List of source directories to backup (required). Globs and tildes are expanded.
+                example:
+                    - /home
+                    - /etc
+                    - /var/log/syslog*
+            one_file_system:
+                type: bool
+                desc: Stay in same file system (do not cross mount points).
+                example: true
+            bsd_flags:
+                type: bool
+                desc: Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true.
+                example: true
+            files_cache:
+                type: scalar
+                desc: |
+                    Mode in which to operate the files cache. See
+                    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
+                    details.
+                example: ctime,size,inode
+            local_path:
+                type: scalar
+                desc: Alternate Borg local executable. Defaults to "borg".
+                example: borg1
+            remote_path:
+                type: scalar
+                desc: Alternate Borg remote executable. Defaults to "borg".
+                example: borg1
+            repositories:
+                required: true
+                seq:
+                    - type: scalar
+                desc: |
+                    Paths to local or remote repositories (required). Tildes are expanded. Multiple
+                    repositories are backed up to in sequence. See ssh_command for SSH options like
+                    identity file or port.
+                example:
+                    - user@backupserver:sourcehostname.borg
+            patterns:
+                seq:
+                    - type: scalar
+                desc: |
+                    Any paths matching these patterns are included/excluded from backups. Globs are
+                    expanded. (Tildes are not.) Note that Borg considers this option experimental.
+                    See the output of "borg help patterns" for more details. Quote any value if it
+                    contains leading punctuation, so it parses correctly.
+                example:
+                    - 'R /'
+                    - '- /home/*/.cache'
+                    - '+ /home/susan'
+                    - '- /home/*'
+            patterns_from:
+                seq:
+                    - type: scalar
+                desc: |
+                    Read include/exclude patterns from one or more separate named files, one pattern
+                    per line. Note that Borg considers this option experimental. See the output of
+                    "borg help patterns" for more details.
+                example:
+                    - /etc/borgmatic/patterns
+            exclude_patterns:
+                seq:
+                    - type: scalar
+                desc: |
+                    Any paths matching these patterns are excluded from backups. Globs and tildes
+                    are expanded. See the output of "borg help patterns" for more details.
+                example:
+                    - '*.pyc'
+                    - ~/*/.cache
+                    - /etc/ssl
+            exclude_from:
+                seq:
+                    - type: scalar
+                desc: |
+                    Read exclude patterns from one or more separate named files, one pattern per
+                    line. See the output of "borg help patterns" for more details.
+                example:
+                    - /etc/borgmatic/excludes
+            exclude_caches:
+                type: bool
+                desc: |
+                    Exclude directories that contain a CACHEDIR.TAG file. See
+                    http://www.brynosaurus.com/cachedir/spec.html for details.
+                example: true
+            exclude_if_present:
+                type: scalar
+                desc: Exclude directories that contain a file with the given filename.
+                example: .nobackup
+    storage:
+        desc: |
+            Repository storage options. See
+            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
+            https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for
+            details.
+        map:
+            encryption_passcommand:
+                type: scalar
+                desc: |
+                    The standard output of this command is used to unlock the encryption key. Only
+                    use on repositories that were initialized with passcommand/repokey encryption.
+                    Note that if both encryption_passcommand and encryption_passphrase are set,
+                    then encryption_passphrase takes precedence.
+                example: "secret-tool lookup borg-repository repo-name"
+            encryption_passphrase:
+                type: scalar
+                desc: |
+                    Passphrase to unlock the encryption key with. Only use on repositories that were
+                    initialized with passphrase/repokey encryption. Quote the value if it contains
+                    punctuation, so it parses correctly. And backslash any quote or backslash
+                    literals as well.
+                example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+            compression:
+                type: scalar
+                desc: |
+                    Type of compression to use when creating archives. See
+                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
+                    Defaults to no compression.
+                example: lz4
+            remote_rate_limit:
+                type: int
+                desc: Remote network upload rate limit in kiBytes/second.
+                example: 100
+            ssh_command:
+                type: scalar
+                desc: Command to use instead of just "ssh". This can be used to specify ssh options.
+                example: ssh -i /path/to/private/key
+            umask:
+                type: scalar
+                desc: Umask to be used for borg create.
+                example: 0077
+            lock_wait:
+                type: int
+                desc: Maximum seconds to wait for acquiring a repository/cache lock.
+                example: 5
+            archive_name_format:
+                type: scalar
+                desc: |
+                    Name of the archive. Borg placeholders can be used. See the output of
+                    "borg help placeholders" for details. Default is
+                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
+                    also specify a prefix in the retention section to avoid accidental pruning of
+                    archives with a different archive name format. And you should also specify a
+                    prefix in the consistency section as well.
+                example: "{hostname}-documents-{now}"
+    retention:
+        desc: |
+            Retention policy for how many backups to keep in each category. See
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
+            At least one of the "keep" options is required for pruning to work.
+        map:
+            keep_within:
+                type: scalar
+                desc: Keep all archives within this time interval.
+                example: 3H
+            keep_minutely:
+                type: int
+                desc: Number of minutely archives to keep.
+                example: 60
+            keep_hourly:
+                type: int
+                desc: Number of hourly archives to keep.
+                example: 24
+            keep_daily:
+                type: int
+                desc: Number of daily archives to keep.
+                example: 7
+            keep_weekly:
+                type: int
+                desc: Number of weekly archives to keep.
+                example: 4
+            keep_monthly:
+                type: int
+                desc: Number of monthly archives to keep.
+                example: 6
+            keep_yearly:
+                type: int
+                desc: Number of yearly archives to keep.
+                example: 1
+            prefix:
+                type: scalar
+                desc: |
+                    When pruning, only consider archive names starting with this prefix.
+                    Borg placeholders can be used. See the output of "borg help placeholders" for
+                    details. Default is "{hostname}-".
+                example: sourcehostname
+    consistency:
+        desc: |
+            Consistency checks to run after backups. See
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
+            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
+        map:
+            checks:
+                seq:
+                    - type: str
+                      enum: ['repository', 'archives', 'extract', 'disabled']
+                      unique: true
+                desc: |
+                    List of one or more consistency checks to run: "repository", "archives", and/or
+                    "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
+                    all consistency checks. "repository" checks the consistency of the repository,
+                    "archive" checks all of the archives, and "extract" does an extraction dry-run
+                    of just the most recent archive.
+                example:
+                    - repository
+                    - archives
+            check_last:
+                type: int
+                desc: Restrict the number of checked archives to the last n. Applies only to the
+                      "archives" check.
+                example: 3
+            prefix:
+                type: scalar
+                desc: |
+                    When performing the "archives" check, only consider archive names starting with
+                    this prefix. Borg placeholders can be used. See the output of
+                    "borg help placeholders" for details. Default is "{hostname}-".
+                example: sourcehostname
+    hooks:
+        desc: |
+            Shell commands or scripts to execute before and after a backup or if an error has occurred.
+            IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
+            Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
+            prevent potential shell injection or privilege escalation.
+        map:
+            before_backup:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute before creating a backup.
+                example:
+                    - echo "`date` - Starting a backup job."
+            after_backup:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute after creating a backup.
+                example:
+                    - echo "`date` - Backup created."
+            on_error:
+                seq:
+                    - type: scalar
+                desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
+                example:
+                    - echo "`date` - Error while creating a backup."

borgmatic/config/validate.py

@@ -0,0 +1,96 @@
+import logging
+import sys
+import warnings
+
+import pkg_resources
+import pykwalify.core
+import pykwalify.errors
+from ruamel import yaml
+
+
+logger = logging.getLogger(__name__)
+
+
+def schema_filename():
+    '''
+    Path to the installed YAML configuration schema file, used to validate and parse the
+    configuration.
+    '''
+    return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
+
+
+class Validation_error(ValueError):
+    '''
+    A collection of error message strings generated when attempting to validate a particular
+    configurartion file.
+    '''
+    def __init__(self, config_filename, error_messages):
+        self.config_filename = config_filename
+        self.error_messages = error_messages
+
+    def __str__(self):
+        '''
+        Render a validation error as a user-facing string.
+        '''
+        return 'An error occurred while parsing a configuration file at {}:\n'.format(
+            self.config_filename
+        ) + '\n'.join(self.error_messages)
+
+
+def apply_logical_validation(config_filename, parsed_configuration):
+    '''
+    Given a parsed and schematically valid configuration as a data structure of nested dicts (see
+    below), run through any additional logical validation checks. If there are any such validation
+    problems, raise a Validation_error.
+    '''
+    archive_name_format = parsed_configuration.get('storage', {}).get('archive_name_format')
+    prefix = parsed_configuration.get('retention', {}).get('prefix')
+
+    if archive_name_format and not prefix:
+        raise Validation_error(
+            config_filename, (
+                'If you provide an archive_name_format, you must also specify a retention prefix.',
+            )
+        )
+
+    consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
+    if archive_name_format and not consistency_prefix:
+        logger.warning('Since version 1.1.16, if you provide `archive_name_format`, you should also'
+                       ' specify `consistency.prefix`.')
+
+
+def parse_configuration(config_filename, schema_filename):
+    '''
+    Given the path to a config filename in YAML format and the path to a schema filename in
+    pykwalify YAML schema format, return the parsed configuration as a data structure of nested
+    dicts and lists corresponding to the schema. Example return value:
+
+       {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
+       'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
+
+    Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
+    have permissions to read the file, or Validation_error if the config does not match the schema.
+    '''
+    logging.getLogger('pykwalify').setLevel(logging.ERROR)
+
+    try:
+        config = yaml.safe_load(open(config_filename))
+        schema = yaml.safe_load(open(schema_filename))
+    except yaml.error.YAMLError as error:
+        raise Validation_error(config_filename, (str(error),))
+
+    # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
+    # simply remove all examples before passing the schema to pykwalify.
+    for section_name, section_schema in schema['map'].items():
+        for field_name, field_schema in section_schema['map'].items():
+            field_schema.pop('example', None)
+
+    validator = pykwalify.core.Core(source_data=config, schema_data=schema)
+    parsed_result = validator.validate(raise_exception=False)
+
+    if validator.validation_errors:
+        raise Validation_error(config_filename, validator.validation_errors)
+
+    apply_logical_validation(config_filename, parsed_result)
+
+    return parsed_result

borgmatic/signals.py

@@ -0,0 +1,18 @@
+import os
+import signal
+
+
+def _handle_signal(signal_number, frame):  # pragma: no cover
+    '''
+    Send the signal to all processes in borgmatic's process group, which includes child process.
+    '''
+    os.killpg(os.getpgrp(), signal_number)
+
+
+def configure_signals():  # pragma: no cover
+    '''
+    Configure borgmatic's signal handlers to pass relevant signals through to any child processes
+    like Borg. Note that SIGINT gets passed through even without these changes.
+    '''
+    for signal_number in (signal.SIGHUP, signal.SIGTERM, signal.SIGUSR1, signal.SIGUSR2):
+        signal.signal(signal_number, _handle_signal)

borgmatic/tests/builtins.py

@@ -1,11 +0,0 @@
-from flexmock import flexmock
-import sys
-
-
-def builtins_mock():
-    try:
-        # Python 2
-        return flexmock(sys.modules['__builtin__'])
-    except KeyError:
-        # Python 3
-        return flexmock(sys.modules['builtins'])

borgmatic/tests/integration/commands/test_borgmatic.py

@@ -0,0 +1,105 @@
+import os
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.commands import borgmatic as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments()
+
+    assert parser.config_paths == config_paths
+    assert parser.excludes_filename == None
+    assert parser.verbosity is None
+    assert parser.json is False
+
+
+def test_parse_arguments_with_path_arguments_overrides_defaults():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+    assert parser.config_paths == ['myconfig']
+    assert parser.excludes_filename == 'myexcludes'
+    assert parser.verbosity is None
+
+
+def test_parse_arguments_with_multiple_config_paths_parses_as_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
+
+    assert parser.config_paths == ['myconfig', 'otherconfig']
+    assert parser.verbosity is None
+
+
+def test_parse_arguments_with_verbosity_flag_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    parser = module.parse_arguments('--verbosity', '1')
+
+    assert parser.config_paths == config_paths
+    assert parser.excludes_filename == None
+    assert parser.verbosity == 1
+
+
+def test_parse_arguments_with_json_flag_overrides_default():
+    parser = module.parse_arguments('--list', '--json')
+    assert parser.json is True
+
+
+def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments()
+
+    assert parser.prune is True
+    assert parser.create is True
+    assert parser.check is True
+
+
+def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--prune')
+
+    assert parser.prune is True
+    assert parser.create is False
+    assert parser.check is False
+
+
+def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    parser = module.parse_arguments('--create', '--check')
+
+    assert parser.prune is False
+    assert parser.create is True
+    assert parser.check is True
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')
+
+
+def test_parse_arguments_with_json_flag_with_list_or_info_flag_does_not_raise_any_error():
+    module.parse_arguments('--list', '--json')
+    module.parse_arguments('--info', '--json')
+
+
+def test_parse_arguments_with_json_flag_but_no_list_or_info_flag_raises_value_error():
+    with pytest.raises(ValueError):
+        module.parse_arguments('--json')
+
+
+def test_parse_arguments_with_json_flag_and_both_list_and_info_flag_raises_value_error():
+    with pytest.raises(ValueError):
+        module.parse_arguments('--list', '--info', '--json')

borgmatic/tests/integration/commands/test_convert_config.py

@@ -0,0 +1,47 @@
+import os
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.commands import convert_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    parser = module.parse_arguments()
+
+    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
+    assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME
+    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
+
+def test_parse_arguments_with_filename_arguments_overrides_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    parser = module.parse_arguments(
+        '--source-config', 'config',
+        '--source-excludes', 'excludes',
+        '--destination-config', 'config.yaml',
+    )
+
+    assert parser.source_config_filename == 'config'
+    assert parser.source_excludes_filename == 'excludes'
+    assert parser.destination_config_filename == 'config.yaml'
+
+
+def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments()
+
+    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
+    assert parser.source_excludes_filename is None
+    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')

borgmatic/tests/integration/commands/test_generate_config.py

@@ -0,0 +1,12 @@
+from borgmatic.commands import generate_config as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    parser = module.parse_arguments()
+
+    assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
+def test_parse_arguments_with_filename_argument_overrides_defaults():
+    parser = module.parse_arguments('--destination', 'config.yaml')
+
+    assert parser.destination_filename == 'config.yaml'

borgmatic/tests/integration/config/test_generate.py

@@ -0,0 +1,65 @@
+from io import StringIO
+import os
+import sys
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.config import generate as module
+
+
+def test_insert_newline_before_comment_does_not_raise():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Comment',)
+
+    module._insert_newline_before_comment(config, field_name)
+
+
+def test_write_configuration_does_not_raise():
+    flexmock(os.path).should_receive('exists').and_return(False)
+    flexmock(os).should_receive('makedirs')
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').and_return(StringIO())
+    flexmock(os).should_receive('chmod')
+
+    module.write_configuration('config.yaml', {})
+
+
+def test_write_configuration_with_already_existing_file_raises():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    with pytest.raises(FileExistsError):
+        module.write_configuration('config.yaml', {})
+
+
+def test_write_configuration_with_already_existing_directory_does_not_raise():
+    flexmock(os.path).should_receive('exists').and_return(False)
+    flexmock(os).should_receive('makedirs').and_raise(FileExistsError)
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').and_return(StringIO())
+    flexmock(os).should_receive('chmod')
+
+    module.write_configuration('config.yaml', {})
+
+
+def test_add_comments_to_configuration_does_not_raise():
+    # Ensure that it can deal with fields both in the schema and missing from the schema.
+    config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
+    schema = {
+        'map': {
+            'foo': {'desc': 'Foo'},
+            'bar': {'desc': 'Bar'},
+        }
+    }
+
+    module.add_comments_to_configuration(config, schema)
+
+
+def test_generate_sample_configuration_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module).should_receive('write_configuration')
+    flexmock(module).should_receive('_schema_to_sample_configuration')
+
+    module.generate_sample_configuration('config.yaml', 'schema.yaml')

borgmatic/tests/integration/config/test_legacy.py

@@ -1,14 +1,9 @@
-try:
-    # Python 2
-    from cStringIO import StringIO
-except ImportError:
-    # Python 3
-    from io import StringIO
+from io import StringIO
 
 from collections import OrderedDict
 import string
 
-from borgmatic import config as module
+from borgmatic.config import legacy as module
 
 
 def test_parse_section_options_with_punctuation_should_return_section_options():

borgmatic/tests/integration/config/test_validate.py

@@ -0,0 +1,152 @@
+import io
+import string
+import sys
+import os
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.config import validate as module
+
+
+def test_schema_filename_returns_plausable_path():
+    schema_path = module.schema_filename()    
+
+    assert schema_path.endswith('/schema.yaml')
+
+
+def mock_config_and_schema(config_yaml, schema_yaml=None):
+    '''
+    Set up mocks for the given config config YAML string and the schema YAML string, or the default
+    schema if no schema is provided. The idea is that that the code under test consumes these mocks
+    when parsing the configuration.
+    '''
+    config_stream = io.StringIO(config_yaml)
+    if schema_yaml is None:
+        schema_stream = open(module.schema_filename())
+    else:
+        schema_stream = io.StringIO(schema_yaml)
+
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return(config_stream)
+    builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream)
+
+
+def test_parse_configuration_transforms_file_into_mapping():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+                - /etc
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_minutely: 60
+            keep_hourly: 24
+            keep_daily: 7
+
+        consistency:
+            checks:
+                - repository
+                - archives
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60},
+        'consistency': {'checks': ['repository', 'archives']},
+    }
+
+
+def test_parse_configuration_passes_through_quoted_punctuation():
+    escaped_punctuation = string.punctuation.replace('\\', r'\\').replace('"', r'\"')
+
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - "{}.borg"
+        '''.format(escaped_punctuation)
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {
+            'source_directories': ['/home'],
+            'repositories': ['{}.borg'.format(string.punctuation)],
+        },
+    }
+
+
+def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+        ''',
+        '''
+        map:
+            location:
+                required: true
+                map:
+                    source_directories:
+                        required: true
+                        seq:
+                            - type: scalar
+                    repositories:
+                        required: true
+                        seq:
+                            - type: scalar
+        '''
+    )
+
+    module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_missing_config_file():
+    with pytest.raises(FileNotFoundError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_missing_schema_file():
+    mock_config_and_schema('')
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)
+
+    with pytest.raises(FileNotFoundError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_syntax_error():
+    mock_config_and_schema('foo:\nbar')
+
+    with pytest.raises(ValueError):
+        module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_raises_for_validation_error():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories: yes
+            repositories:
+                - hostname.borg
+        '''
+    )
+
+    with pytest.raises(module.Validation_error):
+        module.parse_configuration('config.yaml', 'schema.yaml')

borgmatic/tests/integration/test_command.py

@@ -1,69 +0,0 @@
-import os
-import sys
-
-from flexmock import flexmock
-import pytest
-
-from borgmatic import command as module
-
-
-def test_parse_arguments_with_no_arguments_uses_defaults():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
-    parser = module.parse_arguments()
-
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
-    assert parser.verbosity == None
-
-
-def test_parse_arguments_with_filename_arguments_overrides_defaults():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
-    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
-
-    assert parser.config_filename == 'myconfig'
-    assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity == None
-
-
-def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
-    flexmock(os.path).should_receive('exists').and_return(False)
-
-    parser = module.parse_arguments()
-
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == None
-    assert parser.verbosity == None
-
-
-def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
-    flexmock(os.path).should_receive('exists').and_return(False)
-
-    parser = module.parse_arguments('--excludes', 'myexcludes')
-
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity == None
-
-
-def test_parse_arguments_with_verbosity_flag_overrides_default():
-    flexmock(os.path).should_receive('exists').and_return(True)
-
-    parser = module.parse_arguments('--verbosity', '1')
-
-    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
-    assert parser.verbosity == 1
-
-
-def test_parse_arguments_with_invalid_arguments_exits():
-    flexmock(os.path).should_receive('exists').and_return(True)
-    original_stderr = sys.stderr
-    sys.stderr = sys.stdout
-
-    try:
-        with pytest.raises(SystemExit):
-            module.parse_arguments('--posix-me-harder')
-    finally:
-        sys.stderr = original_stderr

borgmatic/tests/unit/borg/test_check.py

@@ -0,0 +1,290 @@
+from subprocess import STDOUT
+import sys
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.borg import check as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+def insert_subprocess_never():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').never()
+
+
+def test_parse_checks_returns_them_as_tuple():
+    checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
+
+    assert checks == ('foo', 'bar')
+
+
+def test_parse_checks_with_missing_value_returns_defaults():
+    checks = module._parse_checks({})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_blank_value_returns_defaults():
+    checks = module._parse_checks({'checks': []})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_disabled_returns_no_checks():
+    checks = module._parse_checks({'checks': ['disabled']})
+
+    assert checks == ()
+
+
+def test_make_check_flags_with_repository_check_returns_flag():
+    flags = module._make_check_flags(('repository',))
+
+    assert flags == ('--repository-only',)
+
+
+def test_make_check_flags_with_extract_omits_extract_flag():
+    flags = module._make_check_flags(('extract',))
+
+    assert flags == ()
+
+
+def test_make_check_flags_with_default_checks_returns_default_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+
+    assert flags == ('--prefix', module.DEFAULT_PREFIX)
+
+
+def test_make_check_flags_with_all_checks_returns_default_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS + ('extract',))
+
+    assert flags == ('--prefix', module.DEFAULT_PREFIX)
+
+
+def test_make_check_flags_with_archives_check_and_last_includes_last_flag():
+    flags = module._make_check_flags(('archives',), check_last=3)
+
+    assert flags == ('--archives-only', '--last', '3', '--prefix', module.DEFAULT_PREFIX)
+
+
+def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
+    flags = module._make_check_flags(('repository',), check_last=3)
+
+    assert flags == ('--repository-only',)
+
+
+def test_make_check_flags_with_default_checks_and_last_includes_last_flag():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
+
+    assert flags == ('--last', '3', '--prefix', module.DEFAULT_PREFIX)
+
+
+def test_make_check_flags_with_archives_check_and_prefix_includes_prefix_flag():
+    flags = module._make_check_flags(('archives',), prefix='foo-')
+
+    assert flags == ('--archives-only', '--prefix', 'foo-')
+
+
+def test_make_check_flags_with_repository_check_and_prefix_omits_prefix_flag():
+    flags = module._make_check_flags(('repository',), prefix='foo-')
+
+    assert flags == ('--repository-only',)
+
+
+def test_make_check_flags_with_default_checks_and_prefix_includes_prefix_flag():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, prefix='foo-')
+
+    assert flags == ('--prefix', 'foo-')
+
+
+@pytest.mark.parametrize(
+    'checks',
+    (
+        ('repository',),
+        ('archives',),
+        ('repository', 'archives'),
+        ('repository', 'archives', 'other'),
+    ),
+)
+def test_check_archives_calls_borg_with_parameters(checks):
+    check_last = flexmock()
+    consistency_config = {'check_last': check_last}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last, None).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_extract_check_calls_extract_only():
+    checks = ('extract',)
+    check_last = flexmock()
+    consistency_config = {'check_last': check_last}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').never()
+    flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
+    checks = ('repository',)
+    consistency_config = {'check_last': None}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--info'),
+        stdout=None, stderr=STDOUT,
+    )
+
+    module.check_archives(
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
+    checks = ('repository',)
+    consistency_config = {'check_last': None}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--debug'),
+        stdout=None, stderr=STDOUT,
+    )
+
+    module.check_archives(
+        verbosity=VERBOSITY_LOTS,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_without_any_checks_bails():
+    consistency_config = {'check_last': None}
+    flexmock(module).should_receive('_parse_checks').and_return(())
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_local_path_calls_borg_via_local_path():
+    checks = ('repository',)
+    check_last = flexmock()
+    consistency_config = {'check_last': check_last}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last, None).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg1', 'check', 'repo'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+        local_path='borg1',
+    )
+
+
+def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
+    checks = ('repository',)
+    check_last = flexmock()
+    consistency_config = {'check_last': check_last}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last, None).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--remote-path', 'borg1'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+        remote_path='borg1',
+    )
+
+
+def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    checks = ('repository',)
+    check_last = flexmock()
+    consistency_config = {'check_last': check_last}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last, None).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--lock-wait', '5'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={'lock_wait': 5},
+        consistency_config=consistency_config,
+    )
+
+
+def test_check_archives_with_retention_prefix():
+    checks = ('repository',)
+    check_last = flexmock()
+    prefix = 'foo-'
+    consistency_config = {'check_last': check_last, 'prefix': prefix}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last, prefix).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo'),
+        stdout=stdout, stderr=STDOUT,
+    )
+
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+    )

borgmatic/tests/unit/borg/test_create.py

@@ -0,0 +1,686 @@
+import os
+
+from flexmock import flexmock
+
+from borgmatic.borg import create as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def test_initialize_environment_with_passcommand_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize_environment({'encryption_passcommand': 'command'})
+        assert os.environ.get('BORG_PASSCOMMAND') == 'command'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_environment_with_passphrase_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize_environment({'encryption_passphrase': 'pass'})
+        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_environment_with_ssh_command_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize_environment({'ssh_command': 'ssh -C'})
+        assert os.environ.get('BORG_RSH') == 'ssh -C'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_environment_without_configuration_should_not_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize_environment({})
+        assert os.environ.get('BORG_PASSCOMMAND') == None
+        assert os.environ.get('BORG_PASSPHRASE') == None
+        assert os.environ.get('BORG_RSH') == None
+    finally:
+        os.environ = orig_environ
+
+
+def test_expand_directory_with_basic_path_passes_it_through():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
+    flexmock(module.glob).should_receive('glob').and_return([])
+
+    paths = module._expand_directory('foo')
+
+    assert paths == ['foo']
+
+
+def test_expand_directory_with_glob_expands():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
+    flexmock(module.glob).should_receive('glob').and_return(['foo', 'food'])
+
+    paths = module._expand_directory('foo*')
+
+    assert paths == ['foo', 'food']
+
+
+def test_expand_directories_flattens_expanded_directories():
+    flexmock(module).should_receive('_expand_directory').with_args('~/foo').and_return(['/root/foo'])
+    flexmock(module).should_receive('_expand_directory').with_args('bar*').and_return(['bar', 'barf'])
+
+    paths = module._expand_directories(('~/foo', 'bar*'))
+
+    assert paths == ('/root/foo', 'bar', 'barf')
+
+
+def test_expand_directories_considers_none_as_no_directories():
+    paths = module._expand_directories(None)
+
+    assert paths == ()
+
+
+def test_write_pattern_file_does_not_raise():
+    temporary_file = flexmock(
+        name='filename',
+        write=lambda mode: None,
+        flush=lambda: None,
+    )
+    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
+
+    module._write_pattern_file(['exclude'])
+
+
+def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
+    module._write_pattern_file([])
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+def test_make_pattern_flags_includes_pattern_filename_when_given():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns': ['R /', '- /var']},
+        pattern_filename='/tmp/patterns',
+    )
+
+    assert pattern_flags == ('--patterns-from', '/tmp/patterns')
+
+
+def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns_from': ['patterns', 'other']},
+    )
+
+    assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', 'other')
+
+
+def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns_from': ['patterns']},
+        pattern_filename='/tmp/patterns',
+    )
+
+    assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', '/tmp/patterns')
+
+
+def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns_from': None},
+    )
+
+    assert pattern_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_patterns': ['*.pyc', '/var']},
+        exclude_filename='/tmp/excludes',
+    )
+
+    assert exclude_flags == ('--exclude-from', '/tmp/excludes')
+
+
+def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
+
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': ['excludes', 'other']},
+    )
+
+    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other')
+
+
+def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': ['excludes']},
+        exclude_filename='/tmp/excludes',
+    )
+
+    assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
+
+
+def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_from': None},
+    )
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_caches_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_caches': True},
+    )
+
+    assert exclude_flags == ('--exclude-caches',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_caches': False},
+    )
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
+    exclude_flags = module._make_exclude_flags(
+        location_config={'exclude_if_present': 'exclude_me'},
+    )
+
+    assert exclude_flags == ('--exclude-if-present', 'exclude_me')
+
+
+def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
+    exclude_flags = module._make_exclude_flags(location_config={})
+
+    assert exclude_flags == ()
+
+
+DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
+
+
+def test_create_archive_calls_borg_with_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND)
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_patterns_calls_borg_with_patterns():
+    pattern_flags = ('--patterns-from', 'patterns')
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + pattern_flags)
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'patterns': ['pattern'],
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
+    exclude_flags = ('--exclude-from', 'excludes')
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(('exclude',))
+    flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes'))
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
+    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': ['exclude'],
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
+
+    module.create_archive(
+        verbosity=VERBOSITY_SOME,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
+
+    module.create_archive(
+        verbosity=VERBOSITY_LOTS,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--dry-run',))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=True,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_dry_run_and_verbosity_some_calls_borg_without_stats_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--dry-run'))
+
+    module.create_archive(
+        verbosity=VERBOSITY_SOME,
+        dry_run=True,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--dry-run'))
+
+    module.create_archive(
+        verbosity=VERBOSITY_LOTS,
+        dry_run=True,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_compression_calls_borg_with_compression_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'compression': 'rle'},
+    )
+
+
+def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'remote_rate_limit': 100},
+    )
+
+
+def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'one_file_system': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND)
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'bsd_flags': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--nobsdflags',))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'bsd_flags': False,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'files_cache': 'ctime,size',
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_local_path_calls_borg_via_local_path():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg1',) + CREATE_COMMAND[1:])
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        local_path='borg1',
+    )
+
+
+def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        remote_path='borg1',
+    )
+
+
+def test_create_archive_with_umask_calls_borg_with_umask_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'umask': 740},
+    )
+
+
+def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'lock_wait': 5},
+    )
+
+
+def test_create_archive_with_source_directories_glob_expands():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_glob_calls_borg_with_expanded_directories():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={
+            'archive_name_format': 'ARCHIVE_NAME',
+        },
+    )
+
+
+def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([])
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={
+            'archive_name_format': 'Documents_{hostname}-{now}',
+        },
+    )

borgmatic/tests/unit/borg/test_extract.py

@@ -0,0 +1,140 @@
+import sys
+
+from flexmock import flexmock
+
+from borgmatic.borg import extract as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+def insert_subprocess_never():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').never()
+
+
+def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        lock_wait=None,
+    )
+
+
+def test_extract_last_archive_dry_run_without_any_archives_should_bail():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo'),
+        result='\n'.encode('utf-8'),
+    )
+    insert_subprocess_never()
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        lock_wait=None,
+    )
+
+
+def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--info'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=VERBOSITY_SOME,
+        repository='repo',
+        lock_wait=None,
+    )
+
+
+def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--debug'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=VERBOSITY_LOTS,
+        repository='repo',
+        lock_wait=None,
+    )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg1', 'list', '--short', 'repo'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg1', 'extract', '--dry-run', 'repo::archive2'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        lock_wait=None,
+        local_path='borg1',
+    )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        lock_wait=None,
+        remote_path='borg1',
+    )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        lock_wait=5,
+    )

borgmatic/tests/unit/borg/test_hook.py

@@ -0,0 +1,22 @@
+from flexmock import flexmock
+
+from borgmatic.commands import hook as module
+
+
+def test_execute_hook_invokes_each_command():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(':', shell=True).once()
+
+    module.execute_hook([':'], 'config.yaml', 'pre-backup')
+
+
+def test_execute_hook_with_multiple_commands_invokes_each_command():
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(':', shell=True).once()
+    subprocess.should_receive('check_call').with_args('true', shell=True).once()
+
+    module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup')
+
+
+def test_execute_hook_with_empty_commands_does_not_raise():
+    module.execute_hook([], 'config.yaml', 'post-backup')

borgmatic/tests/unit/borg/test_info.py

@@ -0,0 +1,88 @@
+from collections import OrderedDict
+
+from flexmock import flexmock
+
+from borgmatic.borg import info as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_output').with_args(check_call_command, **kwargs).once()
+
+
+INFO_COMMAND = ('borg', 'info', 'repo')
+
+
+def test_display_archives_info_calls_borg_with_parameters():
+    insert_subprocess_mock(INFO_COMMAND)
+
+    module.display_archives_info(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+    )
+
+
+def test_display_archives_info_with_verbosity_some_calls_borg_with_info_parameter():
+    insert_subprocess_mock(INFO_COMMAND + ('--info',))
+
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        verbosity=VERBOSITY_SOME,
+    )
+
+
+def test_display_archives_info_with_verbosity_lots_calls_borg_with_debug_parameter():
+    insert_subprocess_mock(INFO_COMMAND + ('--debug',))
+
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        verbosity=VERBOSITY_LOTS,
+    )
+
+
+def test_display_archives_info_with_json_calls_borg_with_json_parameter():
+    insert_subprocess_mock(INFO_COMMAND + ('--json',))
+
+    module.display_archives_info(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        json=True,
+    )
+
+
+def test_display_archives_info_with_local_path_calls_borg_via_local_path():
+    insert_subprocess_mock(('borg1',) + INFO_COMMAND[1:])
+
+    module.display_archives_info(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        local_path='borg1',
+    )
+
+
+def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_parameters():
+    insert_subprocess_mock(INFO_COMMAND + ('--remote-path', 'borg1'))
+
+    module.display_archives_info(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        remote_path='borg1',
+    )
+
+
+def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    insert_subprocess_mock(INFO_COMMAND + ('--lock-wait', '5'))
+
+    module.display_archives_info(
+        verbosity=None,
+        repository='repo',
+        storage_config=storage_config,
+    )

borgmatic/tests/unit/borg/test_list.py

@@ -0,0 +1,88 @@
+from collections import OrderedDict
+
+from flexmock import flexmock
+
+from borgmatic.borg import list as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_output').with_args(check_call_command, **kwargs).once()
+
+
+LIST_COMMAND = ('borg', 'list', 'repo')
+
+
+def test_list_archives_calls_borg_with_parameters():
+    insert_subprocess_mock(LIST_COMMAND)
+
+    module.list_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+    )
+
+
+def test_list_archives_with_verbosity_some_calls_borg_with_info_parameter():
+    insert_subprocess_mock(LIST_COMMAND + ('--info',))
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        verbosity=VERBOSITY_SOME,
+    )
+
+
+def test_list_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
+    insert_subprocess_mock(LIST_COMMAND + ('--debug',))
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        verbosity=VERBOSITY_LOTS,
+    )
+
+
+def test_list_archives_with_json_calls_borg_with_json_parameter():
+    insert_subprocess_mock(LIST_COMMAND + ('--json',))
+
+    module.list_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        json=True,
+    )
+
+
+def test_list_archives_with_local_path_calls_borg_via_local_path():
+    insert_subprocess_mock(('borg1',) + LIST_COMMAND[1:])
+
+    module.list_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        local_path='borg1',
+    )
+
+
+def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
+    insert_subprocess_mock(LIST_COMMAND + ('--remote-path', 'borg1'))
+
+    module.list_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        remote_path='borg1',
+    )
+
+
+def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
+
+    module.list_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config=storage_config,
+    )

borgmatic/tests/unit/borg/test_prune.py

@@ -0,0 +1,187 @@
+from collections import OrderedDict
+
+from flexmock import flexmock
+
+from borgmatic.borg import prune as module
+from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
+
+
+def insert_subprocess_mock(check_call_command, **kwargs):
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+
+
+BASE_PRUNE_FLAGS = (
+    ('--keep-daily', '1'),
+    ('--keep-weekly', '2'),
+    ('--keep-monthly', '3'),
+)
+
+
+def test_make_prune_flags_returns_flags_from_config_plus_default_prefix():
+    retention_config = OrderedDict(
+        (
+            ('keep_daily', 1),
+            ('keep_weekly', 2),
+            ('keep_monthly', 3),
+        )
+    )
+
+    result = module._make_prune_flags(retention_config)
+
+    assert tuple(result) == BASE_PRUNE_FLAGS + (('--prefix', '{hostname}-'),)
+
+
+def test_make_prune_flags_accepts_prefix_with_placeholders():
+    retention_config = OrderedDict(
+        (
+            ('keep_daily', 1),
+            ('prefix', 'Documents_{hostname}-{now}'),
+        )
+    )
+
+    result = module._make_prune_flags(retention_config)
+
+    expected = (
+        ('--keep-daily', '1'),
+        ('--prefix', 'Documents_{hostname}-{now}'),
+    )
+
+    assert tuple(result) == expected
+
+
+PRUNE_COMMAND = (
+    'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
+)
+
+
+def test_prune_archives_calls_borg_with_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND)
+
+    module.prune_archives(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',))
+
+    module.prune_archives(
+        repository='repo',
+        storage_config={},
+        verbosity=VERBOSITY_SOME,
+        dry_run=False,
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list'))
+
+    module.prune_archives(
+        repository='repo',
+        storage_config={},
+        verbosity=VERBOSITY_LOTS,
+        dry_run=False,
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--dry-run',))
+
+    module.prune_archives(
+        repository='repo',
+        storage_config={},
+        verbosity=None,
+        dry_run=True,
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_local_path_calls_borg_via_local_path():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(('borg1',) + PRUNE_COMMAND[1:])
+
+    module.prune_archives(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        retention_config=retention_config,
+        local_path='borg1',
+    )
+
+
+def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1'))
+
+    module.prune_archives(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        retention_config=retention_config,
+        remote_path='borg1',
+    )
+
+
+def test_prune_archives_with_umask_calls_borg_with_umask_parameters():
+    storage_config = {'umask': '077'}
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--umask', '077'))
+
+    module.prune_archives(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        storage_config=storage_config,
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
+
+    module.prune_archives(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        storage_config=storage_config,
+        retention_config=retention_config,
+    )

borgmatic/tests/unit/commands/test_borgmatic.py

@@ -0,0 +1,52 @@
+import json
+import sys
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.commands import borgmatic
+
+
+def test__run_commands_handles_multiple_json_outputs_in_array():
+    (
+        flexmock(borgmatic)
+        .should_receive('_run_commands_on_repository')
+        .times(3)
+        .replace_with(
+            lambda args, consistency, json_results, local_path, location, remote_path, retention,
+            storage,
+            unexpanded_repository: json_results.append({"whatever": unexpanded_repository})
+        )
+    )
+
+    (
+        flexmock(sys.stdout)
+        .should_call("write")
+        .with_args(
+            json.dumps(
+                json.loads(
+                    '''
+                        [
+                            {"whatever": "fake_repo1"},
+                            {"whatever": "fake_repo2"},
+                            {"whatever": "fake_repo3"}
+                        ]
+                    ''',
+                )
+            )
+        )
+    )
+
+    borgmatic._run_commands(
+        args=flexmock(json=True),
+        consistency=None,
+        local_path=None,
+        location={'repositories': [
+            'fake_repo1',
+            'fake_repo2',
+            'fake_repo3'
+        ]},
+        remote_path=None,
+        retention=None,
+        storage=None,
+    )

borgmatic/tests/unit/config/test_collect.py

@@ -0,0 +1,115 @@
+from flexmock import flexmock
+
+from borgmatic.config import collect as module
+
+
+def test_get_default_config_paths_includes_absolute_user_config_path():
+    flexmock(module.os, environ={'XDG_CONFIG_HOME': None, 'HOME': '/home/user'})
+
+    config_paths = module.get_default_config_paths()
+
+    assert '/home/user/.config/borgmatic/config.yaml' in config_paths
+
+
+def test_get_default_config_paths_prefers_xdg_config_home_for_user_config_path():
+    flexmock(module.os, environ={'XDG_CONFIG_HOME': '/home/user/.etc', 'HOME': '/home/user'})
+
+    config_paths = module.get_default_config_paths()
+
+    assert '/home/user/.etc/borgmatic/config.yaml' in config_paths
+
+
+def test_collect_config_filenames_collects_given_files():
+    config_paths = ('config.yaml', 'other.yaml')
+    flexmock(module.os.path).should_receive('isdir').and_return(False)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == config_paths
+
+
+def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
+    config_paths = ('config.yaml', '/etc/borgmatic.d')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').and_return(True)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
+    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml'])
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == (
+        'config.yaml',
+        '/etc/borgmatic.d/foo.yaml',
+        '/etc/borgmatic.d/baz.yaml',
+    )
+
+
+def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_non_yaml_filenames():
+    config_paths = ('/etc/borgmatic.d',)
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').and_return(True)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False)
+    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar.yaml~', 'baz.txt'])
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('/etc/borgmatic.d/foo.yaml',)
+
+
+def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/etc/borgmatic/config.yaml')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/etc/borgmatic/config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic/config.yaml').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('config.yaml',)
+
+
+def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/etc/borgmatic.d')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('config.yaml',)
+
+
+def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/etc/../etc/borgmatic.d')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/etc/../etc/borgmatic.d').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/../etc/borgmatic.d').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('config.yaml',)
+
+
+def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/my/directory')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/my/directory').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/my/directory').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == config_paths

borgmatic/tests/unit/config/test_convert.py

@@ -0,0 +1,118 @@
+from collections import defaultdict, OrderedDict, namedtuple
+import os
+
+from flexmock import flexmock
+import pytest
+
+from borgmatic.config import convert as module
+
+
+Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
+
+
+def test_convert_section_generates_integer_value_for_integer_type_in_schema():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    source_section_config = OrderedDict([('check_last', '3')])
+    section_schema = {'map': {'check_last': {'type': 'int'}}}
+
+    destination_config = module._convert_section(source_section_config, section_schema)
+
+    assert destination_config == OrderedDict([('check_last', 3)])
+
+
+def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    source_config = Parsed_config(
+        location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
+        storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
+        retention=OrderedDict([('keep_daily', 7)]),
+        consistency=OrderedDict([('checks', 'repository')]),
+    )
+    source_excludes = ['/var']
+    schema = {'map': defaultdict(lambda: {'map': {}})}
+
+    destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)
+
+    assert destination_config == OrderedDict([
+        (
+            'location',
+            OrderedDict([
+                ('source_directories', ['/home']),
+                ('repositories', ['hostname.borg']),
+                ('exclude_patterns', ['/var']),
+            ]),
+        ),
+        ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
+        ('retention', OrderedDict([('keep_daily', 7)])),
+        ('consistency', OrderedDict([('checks', ['repository'])])),
+    ])
+
+
+def test_convert_legacy_parsed_config_splits_space_separated_values():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    source_config = Parsed_config(
+        location=OrderedDict([('source_directories', '/home /etc'), ('repository', 'hostname.borg')]),
+        storage=OrderedDict(),
+        retention=OrderedDict(),
+        consistency=OrderedDict([('checks', 'repository archives')]),
+    )
+    source_excludes = ['/var']
+    schema = {'map': defaultdict(lambda: {'map': {}})}
+
+    destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) 
+
+    assert destination_config == OrderedDict([
+        (
+            'location',
+            OrderedDict([
+                ('source_directories', ['/home', '/etc']),
+                ('repositories', ['hostname.borg']),
+                ('exclude_patterns', ['/var']),
+            ]),
+        ),
+        ('storage', OrderedDict()),
+        ('retention', OrderedDict()),
+        ('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
+    ])
+
+
+def test_guard_configuration_upgraded_raises_when_only_source_config_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
+
+    with pytest.raises(module.LegacyConfigurationNotUpgraded):
+        module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
+
+
+def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
+
+    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
+
+
+def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
+
+    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
+
+
+def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present():
+    flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
+    flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
+
+    module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
+
+
+def test_guard_excludes_filename_omitted_raises_when_filename_provided():
+    with pytest.raises(module.LegacyExcludesFilenamePresent):
+        module.guard_excludes_filename_omitted(excludes_filename='/etc/borgmatic/excludes')
+
+
+def test_guard_excludes_filename_omitted_does_not_raise_when_filename_not_provided():
+    module.guard_excludes_filename_omitted(excludes_filename=None)

borgmatic/tests/unit/config/test_generate.py

@@ -0,0 +1,49 @@
+from collections import OrderedDict
+
+from flexmock import flexmock
+
+from borgmatic.config import generate as module
+
+
+def test_schema_to_sample_configuration_generates_config_with_examples():
+    flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    flexmock(module).should_receive('add_comments_to_configuration')
+    schema = {
+        'map': OrderedDict([
+            (
+                'section1', {
+                    'map': {
+                        'field1': OrderedDict([
+                            ('example', 'Example 1')
+                        ]),
+                    },
+                },
+            ),
+            (
+                'section2', {
+                    'map': OrderedDict([
+                        ('field2', {'example': 'Example 2'}),
+                        ('field3', {'example': 'Example 3'}),
+                    ]),
+                }
+            ),
+        ])
+    }
+
+    config = module._schema_to_sample_configuration(schema)
+
+    assert config == OrderedDict([
+        (
+            'section1',
+            OrderedDict([
+                ('field1', 'Example 1'),
+            ]),
+        ),
+        (
+            'section2',
+            OrderedDict([
+                ('field2', 'Example 2'),
+                ('field3', 'Example 3'),
+            ]),
+        )
+    ])

borgmatic/tests/unit/config/test_legacy.py

@@ -3,7 +3,7 @@ from collections import OrderedDict
 from flexmock import flexmock
 import pytest
 
-from borgmatic import config as module
+from borgmatic.config import legacy as module
 
 
 def test_option_should_create_config_option():

borgmatic/tests/unit/config/test_validate.py

@@ -0,0 +1,73 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.config import validate as module
+
+
+def test_validation_error_str_contains_error_messages_and_config_filename():
+    error = module.Validation_error('config.yaml', ('oops', 'uh oh'))
+
+    result = str(error)
+
+    assert 'config.yaml' in result
+    assert 'oops' in result
+    assert 'uh oh' in result
+
+
+def test_apply_logical_validation_raises_if_archive_name_format_present_without_prefix():
+    with pytest.raises(module.Validation_error):
+        module.apply_logical_validation(
+            'config.yaml',
+            {
+                'storage': {'archive_name_format': '{hostname}-{now}'},
+                'retention': {'keep_daily': 7},
+            },
+        )
+
+def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix():
+    with pytest.raises(module.Validation_error):
+        module.apply_logical_validation(
+            'config.yaml',
+            {
+                'storage': {'archive_name_format': '{hostname}-{now}'},
+                'retention': {'keep_daily': 7},
+                'consistency': {'prefix': '{hostname}-'}
+            },
+        )
+
+
+def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix():
+    logger = flexmock(module.logger)
+    logger.should_receive('warning').once()
+
+    module.apply_logical_validation(
+        'config.yaml',
+        {
+            'storage': {'archive_name_format': '{hostname}-{now}'},
+            'retention': {'prefix': '{hostname}-'},
+            'consistency': {},
+        },
+    )
+
+
+def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
+    logger = flexmock(module.logger)
+    logger.should_receive('warning').never()
+
+    module.apply_logical_validation(
+        'config.yaml',
+        {
+            'storage': {'archive_name_format': '{hostname}-{now}'},
+            'retention': {'prefix': '{hostname}-'},
+            'consistency': {'prefix': '{hostname}-'}
+        },
+    )
+
+
+def test_apply_logical_validation_does_not_raise_otherwise():
+    module.apply_logical_validation(
+        'config.yaml',
+        {
+            'retention': {'keep_secondly': 1000},
+        },
+    )

borgmatic/tests/unit/test_borg.py

@@ -1,420 +0,0 @@
-from collections import OrderedDict
-from subprocess import STDOUT
-import os
-
-from flexmock import flexmock
-
-from borgmatic import borg as module
-from borgmatic.tests.builtins import builtins_mock
-from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
-
-
-def test_initialize_with_passphrase_should_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({'encryption_passphrase': 'pass'}, command='attic')
-        assert os.environ.get('ATTIC_PASSPHRASE') == 'pass'
-    finally:
-        os.environ = orig_environ
-
-
-def test_initialize_without_passphrase_should_not_set_environment():
-    orig_environ = os.environ
-
-    try:
-        os.environ = {}
-        module.initialize({}, command='attic')
-        assert os.environ.get('ATTIC_PASSPHRASE') == None
-    finally:
-        os.environ = orig_environ
-
-
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(STDOUT=STDOUT)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-    flexmock(module).subprocess = subprocess
-
-
-def insert_subprocess_never():
-    subprocess = flexmock()
-    subprocess.should_receive('check_call').never()
-    flexmock(module).subprocess = subprocess
-
-
-def insert_platform_mock():
-    flexmock(module.platform).should_receive('node').and_return('host')
-
-
-def insert_datetime_mock():
-    flexmock(module).datetime = flexmock().should_receive('now').and_return(
-        flexmock().should_receive('isoformat').and_return('now').mock
-    ).mock
-
-
-CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar')
-CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
-
-
-def test_create_archive_should_call_attic_with_parameters():
-    insert_subprocess_mock(CREATE_COMMAND)
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
-        verbosity=None,
-        storage_config={},
-        source_directories='foo bar',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_two_spaces_in_source_directories():
-    insert_subprocess_mock(CREATE_COMMAND)
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
-        verbosity=None,
-        storage_config={},
-        source_directories='foo  bar',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes():
-    insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename=None,
-        verbosity=None,
-        storage_config={},
-        source_directories='foo bar',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
-    insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
-        verbosity=VERBOSITY_SOME,
-        storage_config={},
-        source_directories='foo bar',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter():
-    insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
-        verbosity=VERBOSITY_LOTS,
-        storage_config={},
-        source_directories='foo bar',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_compression_should_call_attic_with_compression_parameters():
-    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
-        verbosity=None,
-        storage_config={'compression': 'rle'},
-        source_directories='foo bar',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters():
-    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
-        verbosity=None,
-        storage_config={},
-        source_directories='foo bar',
-        repository='repo',
-        command='attic',
-        one_file_system=True,
-    )
-
-
-def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
-    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
-        verbosity=None,
-        storage_config={'umask': 740},
-        source_directories='foo bar',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_source_directories_glob_expands():
-    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
-
-    module.create_archive(
-        excludes_filename=None,
-        verbosity=None,
-        storage_config={},
-        source_directories='foo*',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*'))
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return([])
-
-    module.create_archive(
-        excludes_filename=None,
-        verbosity=None,
-        storage_config={},
-        source_directories='foo*',
-        repository='repo',
-        command='attic',
-    )
-
-
-def test_create_archive_with_glob_should_call_attic_with_expanded_directories():
-    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
-    insert_platform_mock()
-    insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
-
-    module.create_archive(
-        excludes_filename=None,
-        verbosity=None,
-        storage_config={},
-        source_directories='foo*',
-        repository='repo',
-        command='attic',
-    )
-
-
-BASE_PRUNE_FLAGS = (
-    ('--keep-daily', '1'),
-    ('--keep-weekly', '2'),
-    ('--keep-monthly', '3'),
-)
-
-
-def test_make_prune_flags_should_return_flags_from_config():
-    retention_config = OrderedDict(
-        (
-            ('keep_daily', 1),
-            ('keep_weekly', 2),
-            ('keep_monthly', 3),
-        )
-    )
-
-    result = module._make_prune_flags(retention_config)
-
-    assert tuple(result) == BASE_PRUNE_FLAGS
-
-
-PRUNE_COMMAND = (
-    'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
-)
-
-
-def test_prune_archives_should_call_attic_with_parameters():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND)
-
-    module.prune_archives(
-        verbosity=None,
-        repository='repo',
-        retention_config=retention_config,
-        command='attic',
-    )
-
-
-def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--stats',))
-
-    module.prune_archives(
-        repository='repo',
-        verbosity=VERBOSITY_SOME,
-        retention_config=retention_config,
-        command='attic',
-    )
-
-
-def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
-    retention_config = flexmock()
-    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
-        BASE_PRUNE_FLAGS,
-    )
-    insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',))
-
-    module.prune_archives(
-        repository='repo',
-        verbosity=VERBOSITY_LOTS,
-        retention_config=retention_config,
-        command='attic',
-    )
-
-
-def test_parse_checks_returns_them_as_tuple():
-    checks = module._parse_checks({'checks': 'foo disabled bar'})
-
-    assert checks == ('foo', 'bar')
-
-
-def test_parse_checks_with_missing_value_returns_defaults():
-    checks = module._parse_checks({})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_blank_value_returns_defaults():
-    checks = module._parse_checks({'checks': ''})
-
-    assert checks == module.DEFAULT_CHECKS
-
-
-def test_parse_checks_with_disabled_returns_no_checks():
-    checks = module._parse_checks({'checks': 'disabled'})
-
-    assert checks == ()
-
-
-def test_make_check_flags_with_checks_returns_flags():
-    flags = module._make_check_flags(('foo', 'bar'))
-
-    assert flags == ('--foo-only', '--bar-only')
-
-
-def test_make_check_flags_with_default_checks_returns_no_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS)
-
-    assert flags == ()
-
-
-def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
-    flags = module._make_check_flags(('foo', 'bar'), check_last=3)
-
-    assert flags == ('--foo-only', '--bar-only', '--last', 3)
-
-
-def test_make_check_flags_with_last_returns_last_flag():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
-
-    assert flags == ('--last', 3)
-
-
-def test_check_archives_should_call_attic_with_parameters():
-    checks = flexmock()
-    check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
-    flexmock(module).should_receive('_parse_checks').and_return(checks)
-    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
-    stdout = flexmock()
-    insert_subprocess_mock(
-        ('attic', 'check', 'repo'),
-        stdout=stdout, stderr=STDOUT,
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-    builtins_mock().should_receive('open').and_return(stdout)
-    flexmock(module.os).should_receive('devnull')
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='attic',
-    )
-
-
-def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('attic', 'check', 'repo', '--verbose'),
-        stdout=None, stderr=STDOUT,
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.check_archives(
-        verbosity=VERBOSITY_SOME,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='attic',
-    )
-
-
-def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
-    flexmock(module).should_receive('_make_check_flags').and_return(())
-    insert_subprocess_mock(
-        ('attic', 'check', 'repo', '--verbose'),
-        stdout=None, stderr=STDOUT,
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.check_archives(
-        verbosity=VERBOSITY_LOTS,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='attic',
-    )
-
-
-def test_check_archives_without_any_checks_should_bail():
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
-    flexmock(module).should_receive('_parse_checks').and_return(())
-    insert_subprocess_never()
-
-    module.check_archives(
-        verbosity=None,
-        repository='repo',
-        consistency_config=consistency_config,
-        command='attic',
-    )

borgmatic/tests/unit/test_verbosity.py

@@ -0,0 +1,11 @@
+import logging
+
+from borgmatic import verbosity as module
+
+
+def test_verbosity_to_log_level_maps_known_verbosity_to_log_level():
+    assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO
+
+
+def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level():
+    assert module.verbosity_to_log_level('my pants') == logging.WARNING

borgmatic/verbosity.py

@@ -1,2 +1,15 @@
+import logging
+
+
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2
+
+
+def verbosity_to_log_level(verbosity):
+    '''
+    Given a borgmatic verbosity value, return the corresponding Python log level.
+    '''
+    return {
+        VERBOSITY_SOME: logging.INFO,
+        VERBOSITY_LOTS: logging.DEBUG,
+    }.get(verbosity, logging.WARNING)

sample/config

@@ -1,42 +0,0 @@
-[location]
-# Space-separated list of source directories to backup.
-# Globs are expanded.
-source_directories: /home /etc /var/log/syslog*
-
-# For Borg only, you can specify to stay in same file system (do not cross
-# mount points).
-#one_file_system: True
-
-# Path to local or remote repository.
-repository: user@backupserver:sourcehostname.borg
-
-[storage]
-# Passphrase to unlock the encryption key with. Only use on repositories that
-# were initialized with passphrase/repokey encryption.
-#encryption_passphrase: foo
-# Type of compression to use when creating archives. See
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
-# for details. Defaults to no compression.
-#compression: lz4
-# Umask to be used for borg create.
-#umask: 0740
-
-[retention]
-# Retention policy for how many backups to keep in each category. See
-# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
-#keep_within: 3H
-#keep_hourly: 24
-keep_daily: 7
-keep_weekly: 4
-keep_monthly: 6
-keep_yearly: 1
-#prefix: sourcehostname
-
-[consistency]
-# Space-separated list of consistency checks to run: "repository", "archives",
-# or both. Defaults to both. Set to "disabled" to disable all consistency
-# checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
-# for details.
-checks: repository archives
-# Restrict the number of checked archives to the last n.
-#check_last: 3

sample/excludes

@@ -1,3 +0,0 @@
-*.pyc
-/home/*/.cache
-/etc/ssl

sample/systemd/borgmatic.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=borgmatic backup
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/borgmatic

sample/systemd/borgmatic.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run borgmatic backup
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+
+[Install]
+WantedBy=timers.target

scripts/find-unsupported-borg-options

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -o nounset
+
+# For each Borg sub-command that borgmatic uses, print out the Borg flags that borgmatic does not
+# appear to support yet. This script isn't terribly robust. It's intended as a basic tool to ferret
+# out unsupported Borg options so that they can be considered for addition to borgmatic.
+
+# Generate a sample borgmatic configuration with all options set.
+generate-borgmatic-config --destination temp.yaml
+
+# For each sub-command (prune, create, and check), collect the Borg command-line flags that result
+# from running borgmatic with the generated configuration. Then, collect the full set of available
+# Borg flags as reported by "borg --help" for that sub-command. Finally, compare the two lists of
+# flags to determine which Borg flags borgmatic doesn't yet support.
+for sub_command in prune create check list info; do
+    echo "********** borg $sub_command **********"
+
+    for line in $(borgmatic --config temp.yaml --$sub_command -v 2 2>&1 | grep "borg\w* $sub_command") ; do
+        echo "$line" | grep '^-' >> borgmatic_borg_flags
+    done
+    sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
+    mv borgmatic_borg_flags.sorted borgmatic_borg_flags
+
+    for line in $(borg $sub_command --help | awk -v RS= '/^usage:/') ; do
+        # Exclude a bunch of flags that borgmatic actually supports, but don't get exercised by the
+        # generated sample config, and also flags that don't make sense to support.
+        echo "$line" | grep -- -- | sed -r 's/(\[|\])//g' \
+            | grep -v '^-h$' \
+            | grep -v '^--archives-only$' \
+            | grep -v '^--repository-only$' \
+            | grep -v '^--stats$' \
+            | grep -v '^--list$' \
+            | grep -v '^--critical$' \
+            | grep -v '^--error$' \
+            | grep -v '^--warning$' \
+            | grep -v '^--info$' \
+            | grep -v '^--debug$' \
+            >> all_borg_flags
+    done
+    sort all_borg_flags > all_borg_flags.sorted
+    mv all_borg_flags.sorted all_borg_flags
+
+    comm -13 borgmatic_borg_flags all_borg_flags
+
+    rm ./*_borg_flags
+done
+
+rm temp.yaml

scripts/push

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+git push -u origin master
+git push -u github master

scripts/release

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+version=$(head --lines=1 NEWS)
+git tag $version
+git push origin $version
+git push github $version
+
+rm -fr dist
+python3 setup.py bdist_wheel
+python3 setup.py sdist
+twine upload -r pypi dist/borgmatic-*.tar.gz
+twine upload -r pypi dist/borgmatic-*-py3-none-any.whl

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.0.0'
+VERSION = '1.2.2'
 
 
 setup(
@@ -11,7 +11,6 @@ setup(
     author='Dan Helfman',
     author_email='witten@torsion.org',
     url='https://torsion.org/borgmatic',
-    download_url='https://torsion.org/hg/borgmatic/archive/%s.tar.gz' % VERSION,
     classifiers=(
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
@@ -24,14 +23,22 @@ setup(
     packages=find_packages(),
     entry_points={
         'console_scripts': [
-            'borgmatic = borgmatic.command:main',
+            'borgmatic = borgmatic.commands.borgmatic:main',
+            'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
+            'generate-borgmatic-config = borgmatic.commands.generate_config:main',
         ]
     },
-    obsoletes=(
+    obsoletes=[
         'atticmatic',
+    ],
+    install_requires=(
+        'pykwalify>=1.6.0',
+        'ruamel.yaml>0.15.0,<0.16.0',
+        'setuptools',
     ),
     tests_require=(
         'flexmock',
         'pytest',
-    )
+    ),
+    include_package_data=True,
 )

static/borgmatic.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" x="0px" y="0px" viewBox="0 0 100 100" enable-background="new 0 0 100 100" xml:space="preserve"><g><circle cx="63.461" cy="59.615" r="5.626"></circle><path d="M81.432,48.015c0.779-10.849-1.758-19.894-7.407-26.293C68.447,15.403,59.915,11.923,50,11.923   c-6.729,0-12.834,1.645-17.81,4.724c-0.405-0.06-0.823-0.073-1.248,0.005C8.128,20.817,0,34.623,0,46.154   c0,6.155,1.856,11.005,5.519,14.417c2.082,1.941,4.461,3.223,6.659,3.921C13.041,74.256,18.345,81,23.077,81h1.438   C30.168,94,44.847,99.614,50,99.614c6.708,0,24.493-7.875,27.38-25.392c4.758-1.888,10.697-7.373,10.697-13.711   C88.077,54.639,85.395,49.855,81.432,48.015z M7.691,46.154c0-3.397,1.238-13.399,14.747-19.083   c-2.043,3.821-3.31,8.234-3.771,13.136c-3.06,2.521-5.577,7.792-6.366,15.873c-0.534-0.316-1.056-0.686-1.538-1.135   C8.725,53.045,7.691,50.088,7.691,46.154z M74.23,67.947c-1.806,0-3.309,1.389-3.451,3.188C69.608,85.928,53.914,92.575,50,92.575   c-3.275,0-14.607-4.474-19.03-14.325c1.001-0.709,2.168-1.25,3.646-1.25c3.809,0,3.847,0,3.847,0c2.126,0,7.692-4.023,7.692-19.494   C46.154,46.241,40.588,42,38.462,42c0,0-0.195,0-3.847,0c-3.65,0-5.566-4-7.692-4h-1.046c0.813-4,2.646-8.756,5.473-11.94   c4.321-4.87,10.771-7.329,18.65-7.329c8.016,0,14.528,2.636,18.835,7.515c4.821,5.462,6.697,13.762,5.424,23.95   c-0.123,0.984,0.183,1.99,0.84,2.734s1.602,1.179,2.595,1.179c1.671,0,3.46,2.562,3.46,6.371   C81.153,64.016,75.754,67.812,74.23,67.947z M30.432,59.615c0-3.377,2.734-6.107,6.106-6.107c3.373,0,6.107,2.73,6.107,6.107   s-2.734,6.107-6.107,6.107C33.166,65.723,30.432,62.992,30.432,59.615z"></path></g></svg>

test_requirements.txt

@@ -1,2 +1,5 @@
 flexmock==0.10.2
-pytest==2.9.1
+pykwalify==1.6.1
+pytest==3.6.3
+pytest-cov==2.5.1
+ruamel.yaml>0.15.0,<0.16.0

tox.ini

@@ -1,8 +1,8 @@
 [tox]
-envlist=py27,py34
+envlist=py3
 skipsdist=True
 
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = py.test []
+commands = py.test --cov-report term-missing:skip-covered --cov=borgmatic borgmatic []