Bump version for release.

Fix link to issue tracker in documentation

.drone.yml

@@ -1,10 +1,10 @@
 ---
 kind: pipeline
-name: python-3-5-alpine-3-10
+name: python-3-6-alpine-3-9
 
 services:
   - name: postgresql
-    image: postgres:11.6-alpine
+    image: postgres:11.9-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -16,29 +16,7 @@ services:
 
 steps:
 - name: build
-  image: python:3.5-alpine3.10
-  pull: always
-  commands:
-    - scripts/run-full-tests
----
-kind: pipeline
-name: python-3-6-alpine-3-10
-
-services:
-  - name: postgresql
-    image: postgres:11.6-alpine
-    environment:
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: test
-  - name: mysql
-    image: mariadb:10.3
-    environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-
-steps:
-- name: build
-  image: python:3.6-alpine3.10
+  image: alpine:3.9
   pull: always
   commands:
     - scripts/run-full-tests
@@ -48,7 +26,7 @@ name: python-3-7-alpine-3-10
 
 services:
   - name: postgresql
-    image: postgres:11.6-alpine
+    image: postgres:11.9-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -60,51 +38,29 @@ services:
 
 steps:
 - name: build
-  image: python:3.7-alpine3.10
+  image: alpine:3.10
   pull: always
   commands:
     - scripts/run-full-tests
 ---
 kind: pipeline
-name: python-3-7-alpine-3-7
+name: python-3-8-alpine-3-13
 
 services:
   - name: postgresql
-    image: postgres:10.11-alpine
+    image: postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   - name: mysql
-    image: mariadb:10.1
+    image: mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
 
 steps:
 - name: build
-  image: python:3.7-alpine3.7
-  pull: always
-  commands:
-    - scripts/run-full-tests
----
-kind: pipeline
-name: python-3-8-alpine-3-10
-
-services:
-  - name: postgresql
-    image: postgres:11.6-alpine
-    environment:
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: test
-  - name: mysql
-    image: mariadb:10.3
-    environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-
-steps:
-- name: build
-  image: python:3.8-alpine3.10
+  image: alpine:3.13
   pull: always
   commands:
     - scripts/run-full-tests
@@ -122,6 +78,7 @@ steps:
       from_secret: docker_password
     repo: witten/borgmatic-docs
     dockerfile: docs/Dockerfile
-  when:
-    branch:
-      - master
+
+trigger:
+  branch:
+    - master

.gitignore

@@ -2,7 +2,7 @@
 *.pyc
 *.swp
 .cache
-.coverage
+.coverage*
 .pytest_cache
 .tox
 __pycache__

NEWS

@@ -1,4 +1,35 @@
-1.5.11.dev0
+1.5.15
+ * #419: Document use case of running backups conditionally based on laptop power level:
+   https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
+ * #425: Run arbitrary Borg commands with new "borgmatic borg" action. See the documentation for
+   more information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/
+
+1.5.14
+ * #390: Add link to Hetzner storage offering from the documentation.
+ * #398: Clarify canonical home of borgmatic in documentation.
+ * #406: Clarify that spaces in path names should not be backslashed in path names.
+ * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory!
+ * Fix build so as not to attempt to build and push documentation for a non-master branch.
+ * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13.
+ * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama.
+   IRC connection info: https://torsion.org/borgmatic/#issues
+
+1.5.13
+ * #373: Document that passphrase is used for Borg keyfile encryption, not just repokey encryption.
+ * #404: Add support for ruamel.yaml 0.17.x YAML parsing library.
+ * Update systemd service example to return a permission error when a system call isn't permitted
+   (instead of terminating borgmatic outright).
+ * Drop support for Python 3.5, which has been end-of-lifed.
+ * Add support for Python 3.9.
+ * Update versions of test dependencies (test_requirements.txt and test containers).
+ * Only support black code formatter on Python 3.8+. New black dependencies make installation
+   difficult on older versions of Python.
+ * Replace "improve this documentation" form with link to support and ticket tracker.
+
+1.5.12
+ * Fix for previous release with incorrect version suffix in setup.py. No other changes.
+
+1.5.11
  * #341: Add "temporary_directory" option for changing Borg's temporary directory.
  * #352: Lock down systemd security settings in sample systemd service file.
  * #355: Fix traceback when a database hook value is null in a configuration file.

README.md

@@ -11,6 +11,8 @@ borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
 databases too. Monitor it all with integrated third-party services.
 
+The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
+
 Here's an example configuration file:
 
 ```yaml
@@ -84,10 +86,10 @@ reference guides</a>.
 
 ## Hosting providers
 
-Need somewhere to store your encrypted offsite backups? The following hosting
-providers include specific support for Borg/borgmatic. Using these links and
-services helps support borgmatic development and hosting. (These are referral
-links, but without any tracking scripts or cookies.)
+Need somewhere to store your encrypted off-site backups? The following hosting
+providers include specific support for Borg/borgmatic—and fund borgmatic
+development and hosting when you use these links to sign up. (These are
+referral links, but without any tracking scripts or cookies.)
 
 <ul>
  <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
@@ -95,6 +97,10 @@ links, but without any tracking scripts or cookies.)
  <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
+Additionally, [Hetzner](https://www.hetzner.com/storage/storage-box) has a
+compatible storage offering, but does not currently fund borgmatic
+development or hosting.
+
 ## Support and contributing
 
 ### Issues
@@ -106,9 +112,11 @@ first](https://projects.torsion.org/user/login). Note that you can login with
 an existing GitHub account if you prefer.
 
 If you'd like to chat with borgmatic developers or users, head on over to the
-`#borgmatic` IRC channel on Freenode, either via <a
-href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
-native <a href="irc://chat.freenode.net:6697">IRC client</a>.
+`#borgmatic` IRC channel on Libera Chat, either via <a
+href="https://web.libera.chat/#borgmatic">web chat</a> or a
+native <a href="ircs://irc.libera.chat:6697">IRC client</a>. If you
+don't get a response right away, please hang around a while—or file a ticket
+instead.
 
 Also see the [security
 policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
@@ -120,9 +128,9 @@ Other questions or comments? Contact
 
 ### Contributing
 
-borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://projects.torsion.org/witten/borgmatic), and is also
-mirrored on [GitHub](https://github.com/witten/borgmatic) for convenience.
+borgmatic [source code is
+available](https://projects.torsion.org/witten/borgmatic) and is also mirrored
+on [GitHub](https://github.com/witten/borgmatic) for convenience.
 
 borgmatic is licensed under the GNU General Public License version 3 or any
 later version.

borgmatic/borg/borg.py

@@ -0,0 +1,45 @@
+import logging
+
+from borgmatic.borg.flags import make_flags
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+REPOSITORYLESS_BORG_COMMANDS = {'serve', None}
+
+
+def run_arbitrary_borg(
+    repository, storage_config, options, archive=None, local_path='borg', remote_path=None
+):
+    '''
+    Given a local or remote repository path, a storage config dict, a sequence of arbitrary
+    command-line Borg options, and an optional archive name, run an arbitrary Borg command on the
+    given repository/archive.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    try:
+        options = options[1:] if options[0] == '--' else options
+        borg_command = options[0]
+        command_options = tuple(options[1:])
+    except IndexError:
+        borg_command = None
+        command_options = ()
+
+    repository_archive = '::'.join((repository, archive)) if repository and archive else repository
+
+    full_command = (
+        (local_path,)
+        + ((borg_command,) if borg_command else ())
+        + ((repository_archive,) if borg_command and repository_archive else ())
+        + command_options
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+    )
+
+    return execute_command(
+        full_command, output_log_level=logging.WARNING, borg_local_path=local_path,
+    )

borgmatic/borg/create.py

@@ -220,7 +220,8 @@ def create_archive(
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
 
     full_command = (
-        (local_path, 'create')
+        tuple(local_path.split(' '))
+        + ('create',)
         + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
         + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())

borgmatic/commands/arguments.py

@@ -15,17 +15,18 @@ SUBPARSER_ALIASES = {
     'restore': ['--restore', '-r'],
     'list': ['--list', '-l'],
     'info': ['--info', '-i'],
+    'borg': [],
 }
 
 
 def parse_subparser_arguments(unparsed_arguments, subparsers):
     '''
-    Given a sequence of arguments, and a subparsers object as returned by
-    argparse.ArgumentParser().add_subparsers(), give each requested action's subparser a shot at
-    parsing all arguments. This allows common arguments like "--repository" to be shared across
-    multiple subparsers.
+    Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser
+    instance, give each requested action's subparser a shot at parsing all arguments. This allows
+    common arguments like "--repository" to be shared across multiple subparsers.
 
-    Return the result as a dict mapping from subparser name to a parsed namespace of arguments.
+    Return the result as a tuple of (a dict mapping from subparser name to a parsed namespace of
+    arguments, a list of remaining arguments not claimed by any subparser).
     '''
     arguments = collections.OrderedDict()
     remaining_arguments = list(unparsed_arguments)
@@ -35,7 +36,12 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
         for alias in aliases
     }
 
-    for subparser_name, subparser in subparsers.choices.items():
+    # If the "borg" action is used, skip all other subparsers. This avoids confusion like
+    # "borg list" triggering borgmatic's own list action.
+    if 'borg' in unparsed_arguments:
+        subparsers = {'borg': subparsers['borg']}
+
+    for subparser_name, subparser in subparsers.items():
         if subparser_name not in remaining_arguments:
             continue
 
@@ -47,11 +53,11 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
         parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
         for value in vars(parsed).values():
             if isinstance(value, str):
-                if value in subparsers.choices:
+                if value in subparsers:
                     remaining_arguments.remove(value)
             elif isinstance(value, list):
                 for item in value:
-                    if item in subparsers.choices:
+                    if item in subparsers:
                         remaining_arguments.remove(item)
 
         arguments[canonical_name] = parsed
@@ -59,47 +65,33 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
     # If no actions are explicitly requested, assume defaults: prune, create, and check.
     if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
         for subparser_name in ('prune', 'create', 'check'):
-            subparser = subparsers.choices[subparser_name]
+            subparser = subparsers[subparser_name]
             parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
             arguments[subparser_name] = parsed
 
-    return arguments
-
-
-def parse_global_arguments(unparsed_arguments, top_level_parser, subparsers):
-    '''
-    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
-    object as returned by argparse.ArgumentParser().add_subparsers(), parse and return any global
-    arguments as a parsed argparse.Namespace instance.
-    '''
-    # Ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
-    # are global arguments.
     remaining_arguments = list(unparsed_arguments)
-    present_subparser_names = set()
 
-    for subparser_name, subparser in subparsers.choices.items():
-        if subparser_name not in remaining_arguments:
+    # Now ask each subparser, one by one, to greedily consume arguments.
+    for subparser_name, subparser in subparsers.items():
+        if subparser_name not in arguments.keys():
             continue
 
-        present_subparser_names.add(subparser_name)
+        subparser = subparsers[subparser_name]
         unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
 
-    # If no actions are explicitly requested, assume defaults: prune, create, and check.
-    if (
-        not present_subparser_names
-        and '--help' not in unparsed_arguments
-        and '-h' not in unparsed_arguments
-    ):
-        for subparser_name in ('prune', 'create', 'check'):
-            subparser = subparsers.choices[subparser_name]
-            unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+    # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the
+    # "borg" action.
+    if 'borg' in arguments:
+        borg_options_index = remaining_arguments.index('borg') + 1
+        arguments['borg'].options = remaining_arguments[borg_options_index:]
+        remaining_arguments = remaining_arguments[:borg_options_index]
 
     # Remove the subparser names themselves.
-    for subparser_name in present_subparser_names:
+    for subparser_name, subparser in subparsers.items():
         if subparser_name in remaining_arguments:
             remaining_arguments.remove(subparser_name)
 
-    return top_level_parser.parse_args(remaining_arguments)
+    return (arguments, remaining_arguments)
 
 
 class Extend_action(Action):
@@ -510,8 +502,7 @@ def parse_arguments(*unparsed_arguments):
     )
     list_group = list_parser.add_argument_group('list arguments')
     list_group.add_argument(
-        '--repository',
-        help='Path of repository to list, defaults to the configured repository if there is only one',
+        '--repository', help='Path of repository to list, defaults to the configured repositories',
     )
     list_group.add_argument('--archive', help='Name of archive to list (or "latest")')
     list_group.add_argument(
@@ -601,8 +592,32 @@ def parse_arguments(*unparsed_arguments):
     )
     info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    arguments = parse_subparser_arguments(unparsed_arguments, subparsers)
-    arguments['global'] = parse_global_arguments(unparsed_arguments, top_level_parser, subparsers)
+    borg_parser = subparsers.add_parser(
+        'borg',
+        aliases=SUBPARSER_ALIASES['borg'],
+        help='Run an arbitrary Borg command',
+        description='Run an arbitrary Borg command based on borgmatic\'s configuration',
+        add_help=False,
+    )
+    borg_group = borg_parser.add_argument_group('borg arguments')
+    borg_group.add_argument(
+        '--repository',
+        help='Path of repository to pass to Borg, defaults to the configured repositories',
+    )
+    borg_group.add_argument('--archive', help='Name of archive to pass to Borg (or "latest")')
+    borg_group.add_argument(
+        '--',
+        metavar='OPTION',
+        dest='options',
+        nargs='+',
+        help='Options to pass to Borg, command first ("create", "list", etc). "--" is optional. To specify the repository or the archive, you must use --repository or --archive instead of providing them here.',
+    )
+    borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments, remaining_arguments = parse_subparser_arguments(
+        unparsed_arguments, subparsers.choices
+    )
+    arguments['global'] = top_level_parser.parse_args(remaining_arguments)
 
     if arguments['global'].excludes_filename:
         raise ValueError(

borgmatic/commands/borgmatic.py

@@ -9,6 +9,7 @@ from subprocess import CalledProcessError
 import colorama
 import pkg_resources
 
+from borgmatic.borg import borg as borg_borg
 from borgmatic.borg import check as borg_check
 from borgmatic.borg import create as borg_create
 from borgmatic.borg import environment as borg_environment
@@ -543,6 +544,22 @@ def run_actions(
             )
             if json_output:
                 yield json.loads(json_output)
+    if 'borg' in arguments:
+        if arguments['borg'].repository is None or validate.repositories_match(
+            repository, arguments['borg'].repository
+        ):
+            logger.warning('{}: Running arbitrary Borg command'.format(repository))
+            archive_name = borg_list.resolve_archive_name(
+                repository, arguments['borg'].archive, storage, local_path, remote_path
+            )
+            borg_borg.run_arbitrary_borg(
+                repository,
+                storage,
+                options=arguments['borg'].options,
+                archive=archive_name,
+                local_path=local_path,
+                remote_path=remote_path,
+            )
 
 
 def load_configurations(config_filenames, overrides=None):

borgmatic/config/schema.yaml

@@ -15,11 +15,12 @@ map:
                     - type: str
                 desc: |
                     List of source directories to backup (required). Globs and
-                    tildes are expanded.
+                    tildes are expanded. Do not backslash spaces in path names.
                 example:
                     - /home
                     - /etc
                     - /var/log/syslog*
+                    - /home/user/path with spaces
             repositories:
                 required: true
                 seq:
@@ -123,12 +124,14 @@ map:
                     - type: str
                 desc: |
                     Any paths matching these patterns are excluded from backups.
-                    Globs and tildes are expanded. See the output of "borg help
-                    patterns" for more details.
+                    Globs and tildes are expanded. Do not backslash spaces in
+                    path names. See the output of "borg help patterns" for more
+                    details.
                 example:
                     - '*.pyc'
                     - /home/*/.cache
                     - /etc/ssl
+                    - /home/user/path with spaces
             exclude_from:
                 seq:
                     - type: str
@@ -185,19 +188,20 @@ map:
                 desc: |
                     The standard output of this command is used to unlock the
                     encryption key. Only use on repositories that were
-                    initialized with passcommand/repokey encryption. Note that
-                    if both encryption_passcommand and encryption_passphrase are
-                    set, then encryption_passphrase takes precedence. Defaults
-                    to not set.
+                    initialized with passcommand/repokey/keyfile encryption.
+                    Note that if both encryption_passcommand and
+                    encryption_passphrase are set, then encryption_passphrase
+                    takes precedence. Defaults to not set.
                 example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
                 type: str
                 desc: |
                     Passphrase to unlock the encryption key with. Only use on
-                    repositories that were initialized with passphrase/repokey
-                    encryption. Quote the value if it contains punctuation, so
-                    it parses correctly. And backslash any quote or backslash
-                    literals as well. Defaults to not set.
+                    repositories that were initialized with
+                    passphrase/repokey/keyfile encryption. Quote the value if it
+                    contains punctuation, so it parses correctly. And backslash
+                    any quote or backslash literals as well. Defaults to not
+                    set.
                 example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
             checkpoint_interval:
                 type: int

borgmatic/execute.py

@@ -23,7 +23,7 @@ def exit_code_indicates_error(process, exit_code, borg_local_path=None):
     command = process.args.split(' ') if isinstance(process.args, str) else process.args
 
     if borg_local_path and command[0] == borg_local_path:
-        return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+        return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE)
 
     return bool(exit_code != 0)
 

docs/Dockerfile

@@ -1,13 +1,13 @@
-FROM python:3.8.1-alpine3.11 as borgmatic
+FROM python:3.8-alpine3.12 as borgmatic
 
 COPY . /app
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in init prune create check extract mount umount restore list info; do \
+    && for action in init prune create check extract export-tar mount umount restore list info borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:13.7.0-alpine as html
+FROM node:15.2.1-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -26,7 +26,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.16.1-alpine
+FROM nginx:1.19.4-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/_includes/components/suggestion-form.css

@@ -1,18 +0,0 @@
-#suggestion-form textarea {
-	font-family: sans-serif;
-	width: 100%;
-}
-
-#suggestion-form label {
-	font-weight: bold;
-}
-
-#suggestion-form input[type=email] {
-	font-size: 16px;
-	width: 100%;
-}
-
-#suggestion-form .form-error {
-        color: red;
-}
-

docs/_includes/components/suggestion-form.html

@@ -1,33 +0,0 @@
-<h2>Improve this documentation</h2>
-
-<p>Have an idea on how to make this documentation even better? Send your
-feedback below! But if you need help with borgmatic, or have an idea for a
-borgmatic feature, please use our <a href="https://torsion.org/borgmatic/#issues">issue
-tracker</a> instead.</p>
-
-<form id="suggestion-form">
-    <div><label for="suggestion">Documentation suggestion</label></div>
-    <textarea id="suggestion" rows="8" cols="60" name="suggestion"></textarea>
-    <div data-sk-error="suggestion" class="form-error"></div>
-    <input id="_page" type="hidden" name="_page">
-    <input id="_subject" type="hidden" name="_subject" value="borgmatic documentation suggestion">
-    <br />
-    <label for="email">Email address</label>
-    <div><input id="email" type="email" name="email" placeholder="Only required if you want a response!"></div>
-    <div data-sk-error="email" class="form-error"></div>
-    <br />
-    <div><button type="submit">Send</button></div>
-    <br />
-</form>
-
-<script>
-    document.getElementById('_page').value = window.location.href;
-    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
-  
-    sk('form', 'init', {
-        id: '1d536680ab96',
-        element: '#suggestion-form'
-    });
-</script>
-
-<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/components/suggestion-link.html

@@ -0,0 +1,17 @@
+<h2>Improve this documentation</h2>
+
+<p>Have an idea on how to make this documentation even better? Use our <a
+href="https://projects.torsion.org/witten/borgmatic/issues">issue tracker</a> to send your
+feedback!</p>
+
+<script>
+    document.getElementById('_page').value = window.location.href;
+    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
+  
+    sk('form', 'init', {
+        id: '1d536680ab96',
+        element: '#suggestion-form'
+    });
+</script>
+
+<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/layouts/base.njk

@@ -11,7 +11,6 @@
 {% include 'components/minilink.css' %}
 {% include 'components/toc.css' %}
 {% include 'components/info-blocks.css' %}
-{% include 'components/suggestion-form.css' %}
 {% include 'prism-theme.css' %}
 {% include 'asciinema.css' %}
 {% endset %}

docs/_includes/layouts/main.njk

@@ -28,5 +28,5 @@ headerClass: elv-header-default
 
     {{ content | safe }}
 
-    {% include 'components/suggestion-form.html' %}
+    {% include 'components/suggestion-link.html' %}
 </main>

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -16,9 +16,14 @@ But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
 server is offline, then you'll get an annoying error message and the overall
 borgmatic run will fail (even if individual repositories still complete).
 
+Another variant is when the source machine is only sometimes available for
+backups, e.g. a laptop where you want to skip backups when the battery falls
+below a certain level.
+
 So what if you want borgmatic to swallow the error of a missing drive
-or an offline server, and continue trucking along? That's where the concept of
-"soft failure" come in.
+or an offline server or a low battery—and exit gracefully? That's where the
+concept of "soft failure" come in.
+
 
 ## Soft failure command hooks
 
@@ -78,6 +83,17 @@ hooks:
       - ping -q -c 1 buddys-server.org > /dev/null || exit 75
 ```
 
+Or to only run backups if the battery level is high enough:
+
+```yaml
+hooks:
+    before_backup:
+      - is_battery_percent_at_least.sh 25
+```
+
+(Writing the battery script is left as an exercise to the reader.)
+
+
 ## Caveats and details
 
 There are some caveats you should be aware of with this feature.

docs/how-to/develop-on-borgmatic.md

@@ -3,7 +3,7 @@ title: How to develop on borgmatic
 eleventyNavigation:
   key: Develop on borgmatic
   parent: How-to guides
-  order: 11
+  order: 12
 ---
 ## Source code
 

docs/how-to/make-per-application-backups.md

@@ -140,7 +140,19 @@ What this does is load your configuration files, and for each one, disregard
 the configured value for the `remote_path` option in the `location` section,
 and use the value of `/usr/local/bin/borg1` instead.
 
-Note that the value is parsed as an actual YAML string, so you can even set
+You can even override multiple values at once. For instance:
+
+```bash
+borgmatic create --override section.option1=value1 section.option2=value2
+```
+
+This will accomplish the same thing:
+
+```bash
+borgmatic create --override section.option1=value1 --override section.option2=value2
+```
+
+Note that each value is parsed as an actual YAML string, so you can even set
 list values by using brackets. For instance:
 
 ```bash

docs/how-to/run-arbitrary-borg-commands.md

@@ -0,0 +1,94 @@
+---
+title: How to run arbitrary Borg commands
+eleventyNavigation:
+  key: Run arbitrary Borg commands
+  parent: How-to guides
+  order: 10
+---
+## Running Borg with borgmatic
+
+Borg has several commands (and options) that borgmatic does not currently
+support. Sometimes though, as a borgmatic user, you may find yourself wanting
+to take advantage of these off-the-beaten-path Borg features. You could of
+course drop down to running Borg directly. But then you'd give up all the
+niceties of your borgmatic configuration. You could file a [borgmatic
+ticket](https://torsion.org/borgmatic/#issues) or even a [pull
+request](https://torsion.org/borgmatic/#contributing) to add the feature. But
+what if you need it *now*?
+
+That's where borgmatic's support for running "arbitrary" Borg commands comes
+in. Running Borg commands with borgmatic takes advantage of the following, all
+based on your borgmatic configuration files or command-line arguments:
+
+ * configured repositories (automatically runs your Borg command once for each
+   one)
+ * local and remote Borg binary paths
+ * SSH settings and Borg environment variables
+ * lock wait settings
+ * verbosity
+
+
+### borg action
+
+The way you run Borg with borgmatic is via the `borg` action. Here's a simple
+example:
+
+```bash
+borgmatic borg break-lock
+```
+
+(No `borg` action in borgmatic? Time to upgrade!)
+
+This runs Borg's `break-lock` command once on each configured borgmatic
+repository. Notice how the repository isn't present in the specified Borg
+options, as that part is provided by borgmatic.
+
+You can also specify Borg options for relevant commands:
+
+```bash
+borgmatic borg list --progress
+```
+
+This runs Borg's `list` command once on each configured borgmatic
+repository. However, the native `borgmatic list` action should be preferred
+for most use.
+
+What if you only want to run Borg on a single configured borgmatic repository
+when you've got several configured? Not a problem.
+
+```bash
+borgmatic borg --repository repo.borg break-lock
+```
+
+And what about a single archive?
+
+```bash
+borgmatic borg --archive your-archive-name list
+```
+
+### Limitations
+
+borgmatic's `borg` action is not without limitations:
+
+ * The Borg command you want to run (`create`, `list`, etc.) *must* come first
+   after the `borg` action. If you have any other Borg options to specify,
+   provide them after. For instance, `borgmatic borg list --progress` will work,
+   but `borgmatic borg --progress list` will not.
+ * borgmatic supplies the repository/archive name to Borg for you (based on
+   your borgmatic configuration or the `borgmatic borg --repository`/`--archive`
+   arguments), so do not specify the repository/archive otherwise.
+ * The `borg` action will not currently work for any Borg commands like `borg
+   serve` that do not accept a repository/archive name.
+ * Do not specify any global borgmatic arguments to the right of the `borg`
+   action. (They will be passed to Borg instead of borgmatic.) If you have
+   global borgmatic arguments, specify them *before* the `borg` action.
+ * Unlike other borgmatic actions, you cannot combine the `borg` action with
+   other borgmatic actions. This is to prevent ambiguity in commands like
+   `borgmatic borg list`, in which `list` is both a valid Borg command and a
+   borgmatic action. In this case, only the Borg command is run.
+ * Unlike normal borgmatic actions that support JSON, the `borg` action will
+   not disable certain borgmatic logs to avoid interfering with JSON output.
+
+In general, this `borgmatic borg` feature should be considered an escape
+valve—a feature of second resort. In the long run, it's preferable to wrap
+Borg commands with borgmatic actions that can support them fully.

docs/how-to/set-up-backups.md

@@ -68,7 +68,7 @@ sudo pip3 install --upgrade borgmatic
 
 The main downside of a global install is that borgmatic is less cleanly
 separated from the rest of your Python software, and there's the theoretical
-possibility of libary conflicts. But if you're okay with that, for instance
+possibility of library conflicts. But if you're okay with that, for instance
 on a relatively dedicated system, then a global install can work out fine.
 
 
@@ -93,10 +93,10 @@ installing borgmatic:
 
 ## Hosting providers
 
-Need somewhere to store your encrypted offsite backups? The following hosting
-providers include specific support for Borg/borgmatic. Using these links and
-services helps support borgmatic development and hosting. (These are referral
-links, but without any tracking scripts or cookies.)
+Need somewhere to store your encrypted off-site backups? The following hosting
+providers include specific support for Borg/borgmatic—and fund borgmatic
+development and hosting when you use these links to sign up. (These are
+referral links, but without any tracking scripts or cookies.)
 
 <ul>
  <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
@@ -104,6 +104,10 @@ links, but without any tracking scripts or cookies.)
  <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
+Additionally, [Hetzner](https://www.hetzner.com/storage/storage-box) has a
+compatible storage offering, but does not currently fund borgmatic
+development or hosting.
+
 ## Configuration
 
 After you install borgmatic, generate a sample configuration file:
@@ -139,8 +143,8 @@ configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your
 
 ### Encryption
 
-If you encrypt your Borg repository with a passphrase instead of a key file,
-you'll either need to set the borgmatic `encryption_passphrase` configuration
+If you encrypt your Borg repository with a passphrase or a key file, you'll
+either need to set the borgmatic `encryption_passphrase` configuration
 variable or set the `BORG_PASSPHRASE` environment variable. See the
 [repository encryption
 section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)

docs/how-to/upgrade.md

@@ -3,7 +3,7 @@ title: How to upgrade borgmatic
 eleventyNavigation:
   key: Upgrade borgmatic
   parent: How-to guides
-  order: 10
+  order: 11
 ---
 ## Upgrading
 

sample/systemd/borgmatic.service

@@ -2,14 +2,16 @@
 Description=borgmatic backup
 Wants=network-online.target
 After=network-online.target
+# Prevent borgmatic from running unless the machine is plugged into power. Remove this line if you
+# want to allow borgmatic to run anytime.
 ConditionACPower=true
 
 [Service]
 Type=oneshot
 
-# Security settings for systemd running as root
-# For more details about this settings check the systemd manuals
-# https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+# Security settings for systemd running as root, optional but recommended to improve security. You
+# can disable individual settings if they cause problems for your use case. For more details, see
+# the systemd manual: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
 LockPersonality=true
 # Certain borgmatic features like Healthchecks integration need MemoryDenyWriteExecute to be off.
 # But you can try setting it to "yes" for improved security if you don't use those features.
@@ -29,6 +31,7 @@ RestrictRealtime=yes
 RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
 # Restrict write access
 # Change to 'ProtectSystem=strict' and uncomment 'ProtectHome' to make the whole file
 # system read-only be default and uncomment 'ReadWritePaths' for the required write access.

scripts/release

@@ -15,6 +15,12 @@ if [[ ! -f NEWS ]]; then
 fi
 
 version=$(head --lines=1 NEWS)
+
+if [[ $version =~ .*dev* ]]; then
+    echo "Refusing to release a dev version: $version"
+    exit 1
+fi
+
 git tag $version
 git push origin $version
 git push github $version
@@ -32,7 +38,8 @@ twine upload -r pypi dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
 escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
 curl --silent --request POST \
-    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \
+    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases" \
+    --header "Authorization: token $projects_token" \
     --header "Accept: application/json" \
     --header "Content-Type: application/json" \
     --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}"

scripts/run-full-tests

@@ -10,9 +10,11 @@
 
 set -e
 
-python -m pip install --upgrade pip==20.0.2
-pip install tox==3.14.3
+apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client
+# If certain dependencies of black are available in this version of Alpine, install them.
+apk add --no-cache py3-typed-ast py3-regex || true
+python3 -m pip install --upgrade pip==20.2.4 setuptools==50.3.2
+pip3 install tox==3.20.1
 export COVERAGE_FILE=/tmp/.coverage
-tox --workdir /tmp/.tox
-apk add --no-cache borgbackup postgresql-client mariadb-client
-tox --workdir /tmp/.tox -e end-to-end
+tox --workdir /tmp/.tox --sitepackages
+tox --workdir /tmp/.tox --sitepackages -e end-to-end

setup.cfg

@@ -1,5 +1,5 @@
 [metadata]
-description-file=README.md
+description_file=README.md
 
 [tool:pytest]
 testpaths = tests

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.11.dev0'
+VERSION = '1.5.15'
 
 
 setup(
@@ -32,7 +32,7 @@ setup(
     install_requires=(
         'pykwalify>=1.6.0,<14.06',
         'requests',
-        'ruamel.yaml>0.15.0,<0.17.0',
+        'ruamel.yaml>0.15.0,<0.18.0',
         'setuptools',
         'colorama>=0.4.1,<0.5',
     ),

test_requirements.txt

@@ -1,25 +1,28 @@
-appdirs==1.4.3
-atomicwrites==1.3.0
-attrs==19.3.0
-black==19.3b0; python_version >= '3.6'
-click==7.0
-colorama==0.4.1
-coverage==4.5.4
+appdirs==1.4.4; python_version >= '3.8'
+atomicwrites==1.4.0
+attrs==20.3.0; python_version >= '3.8'
+black==19.10b0; python_version >= '3.8'
+click==7.1.2; python_version >= '3.8'
+colorama==0.4.4
+coverage==5.3
 docopt==0.6.2
-flake8==3.7.9
+flake8==3.8.4
 flexmock==0.10.4
-isort==4.3.21
+isort==5.6.4
 mccabe==0.6.1
-more-itertools==7.2.0
-pluggy==0.13.0
-py==1.8.0
-pycodestyle==2.5.0
-pyflakes==2.1.1
+more-itertools==8.6.0
+pluggy==0.13.1
+pathspec==0.8.1; python_version >= '3.8'
+py==1.10.0
+pycodestyle==2.6.0
+pyflakes==2.2.0
 pykwalify==1.7.0
-pytest==5.2.2
-pytest-cov==2.8.1
-python-dateutil==2.8.0
-PyYAML==5.1.2
-requests==2.22.0
-ruamel.yaml>0.15.0,<0.17.0
-toml==0.10.0
+pytest==6.1.2
+pytest-cov==2.10.1
+python-dateutil==2.8.1
+PyYAML==5.4.1
+regex; python_version >= '3.8'
+requests==2.25.0
+ruamel.yaml>0.15.0,<0.18.0
+toml==0.10.2; python_version >= '3.8'
+typed-ast==1.4.2; python_version >= '3.8'

tests/end-to-end/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:12.2-alpine
+    image: postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -11,7 +11,7 @@ services:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   tests:
-    image: python:3.8-alpine3.11
+    image: alpine:3.13
     volumes:
       - "../..:/app:ro"
     tmpfs:

tests/end-to-end/test_borgmatic.py

@@ -20,6 +20,7 @@ def generate_configuration(config_path, repository_path):
         .read()
         .replace('user@backupserver:sourcehostname.borg', repository_path)
         .replace('- user@backupserver:{fqdn}', '')
+        .replace('- /home/user/path with spaces', '')
         .replace('- /home', '- {}'.format(config_path))
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')

tests/integration/commands/test_arguments.py

@@ -163,6 +163,24 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys):
     assert 'create arguments:' in captured.out
 
 
+def test_parse_arguments_with_action_before_global_options_parses_options():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('prune', '--verbosity', '2')
+
+    assert 'prune' in arguments
+    assert arguments['global'].verbosity == 2
+
+
+def test_parse_arguments_with_global_options_before_action_parses_options():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--verbosity', '2', 'prune')
+
+    assert 'prune' in arguments
+    assert arguments['global'].verbosity == 2
+
+
 def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/test_execute.py

@@ -88,13 +88,21 @@ def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdou
 
 def test_log_outputs_kills_other_processes_when_one_errors():
     flexmock(module.logger).should_receive('log')
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
     flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, None, 'borg'
+    ).and_return(False)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, 2, 'borg'
+    ).and_return(True)
     other_process = subprocess.Popen(
         ['watch', 'true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
     )
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        other_process, None, 'borg'
+    ).and_return(False)
     flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
         process.stdout
     )
@@ -118,10 +126,15 @@ def test_log_outputs_kills_other_processes_when_one_errors():
 def test_log_outputs_truncates_long_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
     flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, None, 'borg'
+    ).and_return(False)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, 2, 'borg'
+    ).and_return(True)
     flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     with pytest.raises(subprocess.CalledProcessError) as error:

tests/unit/borg/test_borg.py

@@ -0,0 +1,123 @@
+import logging
+
+from flexmock import flexmock
+
+from borgmatic.borg import borg as module
+
+from ..test_verbosity import insert_logging_mock
+
+
+def test_run_arbitrary_borg_calls_borg_with_parameters():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--info'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+    insert_logging_mock(logging.INFO)
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--debug', '--show-rc'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+    insert_logging_mock(logging.DEBUG)
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--lock-wait', '5'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config=storage_config, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter():
+    storage_config = {}
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo::archive'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config=storage_config, options=['break-lock'], archive='archive',
+    )
+
+
+def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'], local_path='borg1',
+    )
+
+
+def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'], remote_path='borg1',
+    )
+
+
+def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo', '--progress'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['list', '--progress'],
+    )
+
+
+def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['--', 'break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg',), output_log_level=logging.WARNING, borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=[],
+    )

tests/unit/commands/test_arguments.py

@@ -5,146 +5,137 @@ from borgmatic.commands import arguments as module
 
 def test_parse_subparser_arguments_consumes_subparser_arguments_before_subparser_name():
     action_namespace = flexmock(foo=True)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('--foo', 'true', 'action'), subparsers
     )
 
-    arguments = module.parse_subparser_arguments(('--foo', 'true', 'action'), subparsers)
-
     assert arguments == {'action': action_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_consumes_subparser_arguments_after_subparser_name():
     action_namespace = flexmock(foo=True)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('action', '--foo', 'true'), subparsers
     )
 
-    arguments = module.parse_subparser_arguments(('action', '--foo', 'true'), subparsers)
-
     assert arguments == {'action': action_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_consumes_subparser_arguments_with_alias():
     action_namespace = flexmock(foo=True)
-    action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, []))
-    subparsers = flexmock(
-        choices={
-            'action': action_subparser,
-            '-a': action_subparser,
-            'other': flexmock(),
-            '-o': flexmock(),
-        }
-    )
+    action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, ['action']))
+    subparsers = {
+        'action': action_subparser,
+        '-a': action_subparser,
+        'other': flexmock(),
+        '-o': flexmock(),
+    }
     flexmock(module).SUBPARSER_ALIASES = {'action': ['-a'], 'other': ['-o']}
 
-    arguments = module.parse_subparser_arguments(('-a', '--foo', 'true'), subparsers)
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('-a', '--foo', 'true'), subparsers
+    )
 
     assert arguments == {'action': action_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_consumes_multiple_subparser_arguments():
     action_namespace = flexmock(foo=True)
     other_namespace = flexmock(bar=3)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--bar', '3'])
-            ),
-            'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])),
-        }
-    )
+    subparsers = {
+        'action': flexmock(
+            parse_known_args=lambda arguments: (action_namespace, ['action', '--bar', '3'])
+        ),
+        'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])),
+    }
 
-    arguments = module.parse_subparser_arguments(
+    arguments, remaining_arguments = module.parse_subparser_arguments(
         ('action', '--foo', 'true', 'other', '--bar', '3'), subparsers
     )
 
     assert arguments == {'action': action_namespace, 'other': other_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_applies_default_subparsers():
     prune_namespace = flexmock()
     create_namespace = flexmock(progress=True)
     check_namespace = flexmock()
-    subparsers = flexmock(
-        choices={
-            'prune': flexmock(parse_known_args=lambda arguments: (prune_namespace, ['--progress'])),
-            'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])),
-            'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])),
-            'other': flexmock(),
-        }
-    )
+    subparsers = {
+        'prune': flexmock(
+            parse_known_args=lambda arguments: (prune_namespace, ['prune', '--progress'])
+        ),
+        'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])),
+        'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])),
+        'other': flexmock(),
+    }
 
-    arguments = module.parse_subparser_arguments(('--progress'), subparsers)
+    arguments, remaining_arguments = module.parse_subparser_arguments(('--progress'), subparsers)
 
     assert arguments == {
         'prune': prune_namespace,
         'create': create_namespace,
         'check': check_namespace,
     }
+    assert remaining_arguments == []
 
 
-def test_parse_global_arguments_with_help_does_not_apply_default_subparsers():
-    global_namespace = flexmock(verbosity='lots')
+def test_parse_subparser_arguments_passes_through_unknown_arguments_before_subparser_name():
     action_namespace = flexmock()
-    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
-            ),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(
+            parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots'])
+        ),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('--verbosity', 'lots', 'action'), subparsers
     )
 
-    arguments = module.parse_global_arguments(
-        ('--verbosity', 'lots', '--help'), top_level_parser, subparsers
-    )
-
-    assert arguments == global_namespace
+    assert arguments == {'action': action_namespace}
+    assert remaining_arguments == ['--verbosity', 'lots']
 
 
-def test_parse_global_arguments_consumes_global_arguments_before_subparser_name():
-    global_namespace = flexmock(verbosity='lots')
+def test_parse_subparser_arguments_passes_through_unknown_arguments_after_subparser_name():
     action_namespace = flexmock()
-    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
-            ),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(
+            parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots'])
+        ),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('action', '--verbosity', 'lots'), subparsers
     )
 
-    arguments = module.parse_global_arguments(
-        ('--verbosity', 'lots', 'action'), top_level_parser, subparsers
-    )
-
-    assert arguments == global_namespace
+    assert arguments == {'action': action_namespace}
+    assert remaining_arguments == ['--verbosity', 'lots']
 
 
-def test_parse_global_arguments_consumes_global_arguments_after_subparser_name():
-    global_namespace = flexmock(verbosity='lots')
-    action_namespace = flexmock()
-    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
-            ),
-            'other': flexmock(),
-        }
-    )
+def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparsers():
+    action_namespace = flexmock(options=[])
+    subparsers = {
+        'borg': flexmock(parse_known_args=lambda arguments: (action_namespace, ['borg', 'list'])),
+        'list': flexmock(),
+    }
 
-    arguments = module.parse_global_arguments(
-        ('action', '--verbosity', 'lots'), top_level_parser, subparsers
-    )
+    arguments, remaining_arguments = module.parse_subparser_arguments(('borg', 'list'), subparsers)
 
-    assert arguments == global_namespace
+    assert arguments == {'borg': action_namespace}
+    assert arguments['borg'].options == ['list']
+    assert remaining_arguments == []

tests/unit/commands/test_borgmatic.py

@@ -447,7 +447,9 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
         ['baz']
     )
-    flexmock(module.sys.stdout).should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    stdout = flexmock()
+    stdout.should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    flexmock(module.sys).stdout = stdout
     arguments = {}
 
     tuple(

tests/unit/hooks/test_healthchecks.py

@@ -28,7 +28,9 @@ def test_forgetful_buffering_handler_emit_forgets_log_records_when_capacity_reac
 def test_format_buffered_logs_for_payload_flattens_log_buffer():
     handler = module.Forgetful_buffering_handler(byte_capacity=100, log_level=1)
     handler.buffer = ['foo\n', 'bar\n']
-    flexmock(module.logging).should_receive('getLogger').and_return(flexmock(handlers=[handler]))
+    logger = flexmock(handlers=[handler])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 
@@ -39,7 +41,9 @@ def test_format_buffered_logs_for_payload_inserts_truncation_indicator_when_logs
     handler = module.Forgetful_buffering_handler(byte_capacity=100, log_level=1)
     handler.buffer = ['foo\n', 'bar\n']
     handler.forgot = True
-    flexmock(module.logging).should_receive('getLogger').and_return(flexmock(handlers=[handler]))
+    logger = flexmock(handlers=[handler])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 
@@ -47,9 +51,9 @@ def test_format_buffered_logs_for_payload_inserts_truncation_indicator_when_logs
 
 
 def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload():
-    flexmock(module.logging).should_receive('getLogger').and_return(
-        flexmock(handlers=[module.logging.Handler()])
-    )
+    logger = flexmock(handlers=[module.logging.Handler()])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 

tests/unit/test_execute.py

@@ -21,6 +21,11 @@ from borgmatic import execute as module
         (flexmock(args=['grep']), 0, 'borg', False),
         (flexmock(args=['borg']), 0, 'borg', False),
         (flexmock(args=['borg1']), 0, 'borg1', False),
+        # -9 exit code occurs when child process get SIGKILLed.
+        (flexmock(args=['grep']), -9, None, True),
+        (flexmock(args=['grep']), -9, 'borg', True),
+        (flexmock(args=['borg']), -9, 'borg', True),
+        (flexmock(args=['borg1']), -9, 'borg1', True),
         (flexmock(args=['borg']), None, None, False),
     ),
 )

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35,py36,py37,py38
+envlist = py36,py37,py38,py39
 skip_missing_interpreters = True
 skipsdist = True
 minversion = 3.14.1
@@ -13,8 +13,8 @@ whitelist_externals =
 passenv = COVERAGE_FILE
 commands =
     pytest {posargs}
-    py36,py37,py38: black --check .
-    isort --recursive --check-only --settings-path setup.cfg .
+    py38,py39: black --check .
+    isort --check-only --settings-path setup.cfg .
     flake8 borgmatic tests
 
 [testenv:black]
@@ -34,4 +34,4 @@ commands =
 [testenv:isort]
 deps = {[testenv]deps}
 commands =
-    isort {posargs:--recursive} --settings-path setup.cfg .
+    isort --settings-path setup.cfg .