Add support for ruamel.yaml 0.17.x YAML parsing library (#404).

Fix link to issue tracker in documentation

.drone.yml

@@ -1,10 +1,10 @@
 ---
 kind: pipeline
-name: python-3-5-alpine-3-10
+name: python-3-6-alpine-3-9
 
 services:
   - name: postgresql
-    image: postgres:11.6-alpine
+    image: postgres:11.9-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -16,29 +16,7 @@ services:
 
 steps:
 - name: build
-  image: python:3.5-alpine3.10
-  pull: always
-  commands:
-    - scripts/run-full-tests
----
-kind: pipeline
-name: python-3-6-alpine-3-10
-
-services:
-  - name: postgresql
-    image: postgres:11.6-alpine
-    environment:
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: test
-  - name: mysql
-    image: mariadb:10.3
-    environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-
-steps:
-- name: build
-  image: python:3.6-alpine3.10
+  image: alpine:3.9
   pull: always
   commands:
     - scripts/run-full-tests
@@ -48,7 +26,7 @@ name: python-3-7-alpine-3-10
 
 services:
   - name: postgresql
-    image: postgres:11.6-alpine
+    image: postgres:11.9-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -60,51 +38,29 @@ services:
 
 steps:
 - name: build
-  image: python:3.7-alpine3.10
+  image: alpine:3.10
   pull: always
   commands:
     - scripts/run-full-tests
 ---
 kind: pipeline
-name: python-3-7-alpine-3-7
+name: python-3-8-alpine-edge
 
 services:
   - name: postgresql
-    image: postgres:10.11-alpine
+    image: postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   - name: mysql
-    image: mariadb:10.1
+    image: mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
 
 steps:
 - name: build
-  image: python:3.7-alpine3.7
-  pull: always
-  commands:
-    - scripts/run-full-tests
----
-kind: pipeline
-name: python-3-8-alpine-3-10
-
-services:
-  - name: postgresql
-    image: postgres:11.6-alpine
-    environment:
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: test
-  - name: mysql
-    image: mariadb:10.3
-    environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-
-steps:
-- name: build
-  image: python:3.8-alpine3.10
+  image: alpine:edge
   pull: always
   commands:
     - scripts/run-full-tests

NEWS

@@ -1,3 +1,28 @@
+1.5.13
+ * #373: Document that passphrase is used for Borg keyfile encryption, not just repokey encryption.
+ * #404: Add support for ruamel.yaml 0.17.x YAML parsing library.
+ * Update systemd service example to return a permission error when a system call isn't permitted
+   (instead of terminating borgmatic outright).
+ * Drop support for Python 3.5, which has been end-of-lifed.
+ * Add support for Python 3.9.
+ * Update versions of test dependencies (test_requirements.txt and test containers).
+ * Only support black code formatter on Python 3.8+. New black dependencies make installation
+   difficult on older versions of Python.
+ * Replace "improve this documentation" form with link to support and ticket tracker.
+
+1.5.12
+ * Fix for previous release with incorrect version suffix in setup.py. No other changes.
+
+1.5.11
+ * #341: Add "temporary_directory" option for changing Borg's temporary directory.
+ * #352: Lock down systemd security settings in sample systemd service file.
+ * #355: Fix traceback when a database hook value is null in a configuration file.
+ * #361: Merge override values when specifying the "--override" flag multiple times. The previous
+   behavior was to take the value of the last "--override" flag only.
+ * #367: Fix traceback when upgrading old INI-style configuration with upgrade-borgmatic-config.
+ * #368: Fix signal forwarding from borgmatic to Borg resulting in recursion traceback.
+ * #369: Document support for Borg placeholders in repository names.
+
 1.5.10
  * #347: Add hooks that run for the "extract" action: "before_extract" and "after_extract".
  * #350: Fix traceback when a configuration directory is non-readable due to directory permissions.

README.md

@@ -22,8 +22,9 @@ location:
 
     # Paths of local or remote repositories to backup to.
     repositories:
-        - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
         - 1234@usw-s001.rsync.net:backups.borg
+        - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
+        - user1@scp2.cdn.lima-labs.com:repo
         - /var/lib/backups/local.borg
 
 retention:
@@ -66,9 +67,9 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>
+<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic"><img src="docs/static/rsyncnet.png" alt="rsync.net" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-<img src="docs/static/rsyncnet.png" alt="rsync.net" height="60px" style="margin-bottom:20px;">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 
 
 ## Getting started
@@ -89,8 +90,9 @@ services helps support borgmatic development and hosting. (These are referral
 links, but without any tracking scripts or cookies.)
 
 <ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
- <li>rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
 ## Support and contributing

borgmatic/borg/environment.py

@@ -9,6 +9,7 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
     'encryption_passcommand': 'BORG_PASSCOMMAND',
     'encryption_passphrase': 'BORG_PASSPHRASE',
     'ssh_command': 'BORG_RSH',
+    'temporary_directory': 'TMPDIR',
 }
 
 DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {

borgmatic/commands/arguments.py

@@ -1,5 +1,5 @@
 import collections
-from argparse import ArgumentParser
+from argparse import Action, ArgumentParser
 
 from borgmatic.config import collect
 
@@ -102,6 +102,20 @@ def parse_global_arguments(unparsed_arguments, top_level_parser, subparsers):
     return top_level_parser.parse_args(remaining_arguments)
 
 
+class Extend_action(Action):
+    '''
+    An argparse action to support Python 3.8's "extend" action in older versions of Python.
+    '''
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, self.dest, None)
+
+        if items:
+            items.extend(values)
+        else:
+            setattr(namespace, self.dest, list(values))
+
+
 def parse_arguments(*unparsed_arguments):
     '''
     Given command-line arguments with which this script was invoked, parse the arguments and return
@@ -111,6 +125,7 @@ def parse_arguments(*unparsed_arguments):
     unexpanded_config_paths = collect.get_default_config_paths(expand_home=False)
 
     global_parser = ArgumentParser(add_help=False)
+    global_parser.register('action', 'extend', Extend_action)
     global_group = global_parser.add_argument_group('global arguments')
 
     global_group.add_argument(
@@ -178,6 +193,7 @@ def parse_arguments(*unparsed_arguments):
         metavar='SECTION.OPTION=VALUE',
         nargs='+',
         dest='overrides',
+        action='extend',
         help='One or more configuration file options to override with specified values',
     )
     global_group.add_argument(
@@ -581,7 +597,7 @@ def parse_arguments(*unparsed_arguments):
         help='Show info for first N archives after other filters are applied',
     )
     info_group.add_argument(
-        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+        '--last', metavar='N', help='Show info for last N archives after other filters are applied'
     )
     info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 

borgmatic/commands/borgmatic.py

@@ -658,7 +658,9 @@ def collect_configuration_run_summary_logs(configs, arguments):
 
     if not configs:
         yield from make_error_log_records(
-            '{}: No configuration files found'.format(' '.join(arguments['global'].config_paths))
+            '{}: No valid configuration files found'.format(
+                ' '.join(arguments['global'].config_paths)
+            )
         )
         return
 

borgmatic/commands/convert_config.py

@@ -99,7 +99,9 @@ def main():  # pragma: no cover
         )
 
         generate.write_configuration(
-            args.destination_config_filename, destination_config, mode=source_config_file_mode
+            args.destination_config_filename,
+            generate.render_configuration(destination_config),
+            mode=source_config_file_mode,
         )
 
         display_result(args)

borgmatic/config/generate.py

@@ -99,7 +99,7 @@ def _comment_out_optional_configuration(rendered_config):
     return '\n'.join(lines)
 
 
-def _render_configuration(config):
+def render_configuration(config):
     '''
     Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
     '''
@@ -284,5 +284,5 @@ def generate_sample_configuration(source_filename, destination_filename, schema_
 
     write_configuration(
         destination_filename,
-        _comment_out_optional_configuration(_render_configuration(destination_config)),
+        _comment_out_optional_configuration(render_configuration(destination_config)),
     )

borgmatic/config/schema.yaml

@@ -27,10 +27,14 @@ map:
                 desc: |
                     Paths to local or remote repositories (required). Tildes are
                     expanded. Multiple repositories are backed up to in
-                    sequence. See ssh_command for SSH options like identity file
-                    or port.
+                    sequence. Borg placeholders can be used. See the output of
+                    "borg help placeholders" for details. See ssh_command for
+                    SSH options like identity file or port. If systemd service
+                    is used, then add local repository paths in the systemd
+                    service file to the ReadWritePaths list.
                 example:
                     - user@backupserver:sourcehostname.borg
+                    - "user@backupserver:{fqdn}"
             one_file_system:
                 type: bool
                 desc: |
@@ -181,19 +185,20 @@ map:
                 desc: |
                     The standard output of this command is used to unlock the
                     encryption key. Only use on repositories that were
-                    initialized with passcommand/repokey encryption. Note that
-                    if both encryption_passcommand and encryption_passphrase are
-                    set, then encryption_passphrase takes precedence. Defaults
-                    to not set.
+                    initialized with passcommand/repokey/keyfile encryption.
+                    Note that if both encryption_passcommand and
+                    encryption_passphrase are set, then encryption_passphrase
+                    takes precedence. Defaults to not set.
                 example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
                 type: str
                 desc: |
                     Passphrase to unlock the encryption key with. Only use on
-                    repositories that were initialized with passphrase/repokey
-                    encryption. Quote the value if it contains punctuation, so
-                    it parses correctly. And backslash any quote or backslash
-                    literals as well. Defaults to not set.
+                    repositories that were initialized with
+                    passphrase/repokey/keyfile encryption. Quote the value if it
+                    contains punctuation, so it parses correctly. And backslash
+                    any quote or backslash literals as well. Defaults to not
+                    set.
                 example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
             checkpoint_interval:
                 type: int
@@ -226,6 +231,12 @@ map:
                     Remote network upload rate limit in kiBytes/second. Defaults
                     to unlimited.
                 example: 100
+            temporary_directory:
+                type: str
+                desc: |
+                    Directory where temporary files are stored. Defaults to
+                    $TMPDIR
+                example: /path/to/tmpdir
             ssh_command:
                 type: str
                 desc: |

borgmatic/hooks/dispatch.py

@@ -58,5 +58,5 @@ def call_hooks(function_name, hooks, log_prefix, hook_names, *args, **kwargs):
     return {
         hook_name: call_hook(function_name, hooks, log_prefix, hook_name, *args, **kwargs)
         for hook_name in hook_names
-        if hook_name in hooks
+        if hooks.get(hook_name)
     }

borgmatic/signals.py

@@ -4,8 +4,13 @@ import signal
 
 def _handle_signal(signal_number, frame):  # pragma: no cover
     '''
-    Send the signal to all processes in borgmatic's process group, which includes child process.
+    Send the signal to all processes in borgmatic's process group, which includes child processes.
     '''
+    # Prevent infinite signal handler recursion. If the parent frame is this very same handler
+    # function, we know we're recursing.
+    if frame.f_back.f_code.co_name == _handle_signal.__name__:
+        return
+
     os.killpg(os.getpgrp(), signal_number)
 
 

docs/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.1-alpine3.11 as borgmatic
+FROM python:3.8-alpine3.12 as borgmatic
 
 COPY . /app
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
@@ -7,7 +7,7 @@ RUN borgmatic --help > /command-line.txt \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:13.7.0-alpine as html
+FROM node:15.2.1-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -26,7 +26,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.16.1-alpine
+FROM nginx:1.19.4-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/_includes/components/suggestion-form.css

@@ -1,18 +0,0 @@
-#suggestion-form textarea {
-	font-family: sans-serif;
-	width: 100%;
-}
-
-#suggestion-form label {
-	font-weight: bold;
-}
-
-#suggestion-form input[type=email] {
-	font-size: 16px;
-	width: 100%;
-}
-
-#suggestion-form .form-error {
-        color: red;
-}
-

docs/_includes/components/suggestion-form.html

@@ -1,33 +0,0 @@
-<h2>Improve this documentation</h2>
-
-<p>Have an idea on how to make this documentation even better? Send your
-feedback below! But if you need help with borgmatic, or have an idea for a
-borgmatic feature, please use our <a href="https://torsion.org/borgmatic/#issues">issue
-tracker</a> instead.</p>
-
-<form id="suggestion-form">
-    <div><label for="suggestion">Documentation suggestion</label></div>
-    <textarea id="suggestion" rows="8" cols="60" name="suggestion"></textarea>
-    <div data-sk-error="suggestion" class="form-error"></div>
-    <input id="_page" type="hidden" name="_page">
-    <input id="_subject" type="hidden" name="_subject" value="borgmatic documentation suggestion">
-    <br />
-    <label for="email">Email address</label>
-    <div><input id="email" type="email" name="email" placeholder="Only required if you want a response!"></div>
-    <div data-sk-error="email" class="form-error"></div>
-    <br />
-    <div><button type="submit">Send</button></div>
-    <br />
-</form>
-
-<script>
-    document.getElementById('_page').value = window.location.href;
-    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
-  
-    sk('form', 'init', {
-        id: '1d536680ab96',
-        element: '#suggestion-form'
-    });
-</script>
-
-<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/components/suggestion-link.html

@@ -0,0 +1,17 @@
+<h2>Improve this documentation</h2>
+
+<p>Have an idea on how to make this documentation even better? Use our <a
+href="https://projects.torsion.org/witten/borgmatic/issues">issue tracker</a> to send your
+feedback!</p>
+
+<script>
+    document.getElementById('_page').value = window.location.href;
+    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
+  
+    sk('form', 'init', {
+        id: '1d536680ab96',
+        element: '#suggestion-form'
+    });
+</script>
+
+<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/layouts/base.njk

@@ -11,7 +11,6 @@
 {% include 'components/minilink.css' %}
 {% include 'components/toc.css' %}
 {% include 'components/info-blocks.css' %}
-{% include 'components/suggestion-form.css' %}
 {% include 'prism-theme.css' %}
 {% include 'asciinema.css' %}
 {% endset %}

docs/_includes/layouts/main.njk

@@ -28,5 +28,5 @@ headerClass: elv-header-default
 
     {{ content | safe }}
 
-    {% include 'components/suggestion-form.html' %}
+    {% include 'components/suggestion-link.html' %}
 </main>

docs/how-to/backup-your-databases.md

@@ -74,6 +74,17 @@ hooks:
 Note that you may need to use a `username` of the `postgres` superuser for
 this to work with PostgreSQL.
 
+If you would like to backup databases only and not source directories, you can
+specify an empty `source_directories` value because it is a mandatory field:
+
+```yaml
+location:
+    source_directories: []
+hooks:
+    mysql_databases:
+        - name: all
+```
+
 
 ### Configuration backups
 

docs/how-to/deal-with-very-large-backups.md

@@ -18,7 +18,7 @@ repositories.
 
 If you find yourself in this situation, you have some options. First, you can
 run borgmatic's pruning, creating, or checking actions separately. For
-instance, the the following optional actions are available:
+instance, the following optional actions are available:
 
 ```bash
 borgmatic prune

docs/how-to/make-backups-redundant.md

@@ -20,8 +20,9 @@ location:
 
     # Paths of local or remote repositories to backup to.
     repositories:
-        - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
         - 1234@usw-s001.rsync.net:backups.borg
+        - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
+        - user1@scp2.cdn.lima-labs.com:repo
         - /var/lib/backups/local.borg
 ```
 
@@ -32,9 +33,10 @@ your source directories.
 
 Here's a way of visualizing what borgmatic does with the above configuration:
 
-2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
 1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg`
-3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
+2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo`
+3. Backup `/home` and `/etc` to `user1@scp2.cdn.lima-labs.com:repo`
+4. Backup `/home` and `/etc` to `/var/lib/backups/local.borg`
 
 This gives you redundancy of your data across repositories and even
 potentially across providers.

docs/how-to/make-per-application-backups.md

@@ -140,7 +140,19 @@ What this does is load your configuration files, and for each one, disregard
 the configured value for the `remote_path` option in the `location` section,
 and use the value of `/usr/local/bin/borg1` instead.
 
-Note that the value is parsed as an actual YAML string, so you can even set
+You can even override multiple values at once. For instance:
+
+```bash
+borgmatic create --override section.option1=value1 section.option2=value2
+```
+
+This will accomplish the same thing:
+
+```bash
+borgmatic create --override section.option1=value1 --override section.option2=value2
+```
+
+Note that each value is parsed as an actual YAML string, so you can even set
 list values by using brackets. For instance:
 
 ```bash

docs/how-to/set-up-backups.md

@@ -99,8 +99,9 @@ services helps support borgmatic development and hosting. (These are referral
 links, but without any tracking scripts or cookies.)
 
 <ul>
+ <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
- <li>rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
+ <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
 ## Configuration
@@ -138,8 +139,8 @@ configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your
 
 ### Encryption
 
-If you encrypt your Borg repository with a passphrase instead of a key file,
-you'll either need to set the borgmatic `encryption_passphrase` configuration
+If you encrypt your Borg repository with a passphrase or a key file, you'll
+either need to set the borgmatic `encryption_passphrase` configuration
 variable or set the `BORG_PASSPHRASE` environment variable. See the
 [repository encryption
 section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)
@@ -268,6 +269,10 @@ sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/
 sudo systemctl enable --now borgmatic.timer
 ```
 
+Review the security settings in the service file and update them as needed.
+If `ProtectSystem=strict` is enabled and local repositories are used, then
+the repository path must be added to the `ReadWritePaths` list.
+
 Feel free to modify the timer file based on how frequently you'd like
 borgmatic to run.
 

sample/systemd/borgmatic.service

@@ -7,6 +7,39 @@ ConditionACPower=true
 [Service]
 Type=oneshot
 
+# Security settings for systemd running as root, optional but recommended to improve security. You
+# can disable individual settings if they cause problems for your use case. For more details, see
+# the systemd manual: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+LockPersonality=true
+# Certain borgmatic features like Healthchecks integration need MemoryDenyWriteExecute to be off.
+# But you can try setting it to "yes" for improved security if you don't use those features.
+MemoryDenyWriteExecute=no
+NoNewPrivileges=yes
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectClock=yes
+ProtectControlGroups=yes
+ProtectHostname=yes
+ProtectKernelLogs=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+# Restrict write access
+# Change to 'ProtectSystem=strict' and uncomment 'ProtectHome' to make the whole file
+# system read-only be default and uncomment 'ReadWritePaths' for the required write access.
+# Add local repositroy paths to the list of 'ReadWritePaths' like '-/mnt/my_backup_drive'.
+ProtectSystem=full
+# ProtectHome=read-only
+# ReadWritePaths=-/root/.config/borg -/root/.cache/borg -/root/.borgmatic
+
+CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW
+
 # Lower CPU and I/O priority.
 Nice=19
 CPUSchedulingPolicy=batch

scripts/release

@@ -15,6 +15,12 @@ if [[ ! -f NEWS ]]; then
 fi
 
 version=$(head --lines=1 NEWS)
+
+if [[ $version =~ .*dev* ]]; then
+    echo "Refusing to release a dev version: $version"
+    exit 1
+fi
+
 git tag $version
 git push origin $version
 git push github $version

scripts/run-full-tests

@@ -10,9 +10,11 @@
 
 set -e
 
-python -m pip install --upgrade pip==20.0.2
-pip install tox==3.14.3
+apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client
+# If certain dependencies of black are available in this version of Alpine, install them.
+apk add --no-cache py3-typed-ast py3-regex || true
+python3 -m pip install --upgrade pip==20.2.4 setuptools==50.3.2
+pip3 install tox==3.20.1
 export COVERAGE_FILE=/tmp/.coverage
-tox --workdir /tmp/.tox
-apk add --no-cache borgbackup postgresql-client mariadb-client
-tox --workdir /tmp/.tox -e end-to-end
+tox --workdir /tmp/.tox --sitepackages
+tox --workdir /tmp/.tox --sitepackages -e end-to-end

setup.cfg

@@ -1,5 +1,5 @@
 [metadata]
-description-file=README.md
+description_file=README.md
 
 [tool:pytest]
 testpaths = tests

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.10'
+VERSION = '1.5.13'
 
 
 setup(
@@ -32,7 +32,7 @@ setup(
     install_requires=(
         'pykwalify>=1.6.0,<14.06',
         'requests',
-        'ruamel.yaml>0.15.0,<0.17.0',
+        'ruamel.yaml>0.15.0,<0.18.0',
         'setuptools',
         'colorama>=0.4.1,<0.5',
     ),

test_requirements.txt

@@ -1,25 +1,28 @@
-appdirs==1.4.3
-atomicwrites==1.3.0
-attrs==19.3.0
-black==19.3b0; python_version >= '3.6'
-click==7.0
-colorama==0.4.1
-coverage==4.5.4
+appdirs==1.4.4; python_version >= '3.8'
+atomicwrites==1.4.0
+attrs==20.3.0; python_version >= '3.8'
+black==19.10b0; python_version >= '3.8'
+click==7.1.2; python_version >= '3.8'
+colorama==0.4.4
+coverage==5.3
 docopt==0.6.2
-flake8==3.7.9
+flake8==3.8.4
 flexmock==0.10.4
-isort==4.3.21
+isort==5.6.4
 mccabe==0.6.1
-more-itertools==7.2.0
-pluggy==0.13.0
-py==1.8.0
-pycodestyle==2.5.0
-pyflakes==2.1.1
+more-itertools==8.6.0
+pluggy==0.13.1
+pathspec==0.8.1; python_version >= '3.8'
+py==1.9.0
+pycodestyle==2.6.0
+pyflakes==2.2.0
 pykwalify==1.7.0
-pytest==5.2.2
-pytest-cov==2.8.1
-python-dateutil==2.8.0
-PyYAML==5.1.2
-requests==2.22.0
-ruamel.yaml>0.15.0,<0.17.0
-toml==0.10.0
+pytest==6.1.2
+pytest-cov==2.10.1
+python-dateutil==2.8.1
+PyYAML==5.3.1
+regex; python_version >= '3.8'
+requests==2.25.0
+ruamel.yaml>0.15.0,<0.18.0
+toml==0.10.2; python_version >= '3.8'
+typed-ast==1.4.2; python_version >= '3.8'

tests/end-to-end/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:12.2-alpine
+    image: postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -11,7 +11,7 @@ services:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   tests:
-    image: python:3.8-alpine3.11
+    image: alpine:edge
     volumes:
       - "../..:/app:ro"
     tmpfs:

tests/end-to-end/test_borgmatic.py

@@ -19,6 +19,7 @@ def generate_configuration(config_path, repository_path):
         open(config_path)
         .read()
         .replace('user@backupserver:sourcehostname.borg', repository_path)
+        .replace('- user@backupserver:{fqdn}', '')
         .replace('- /home', '- {}'.format(config_path))
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')

tests/integration/commands/test_arguments.py

@@ -71,6 +71,35 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default():
     assert global_arguments.log_file_verbosity == -1
 
 
+def test_parse_arguments_with_single_override_parses():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--override', 'foo.bar=baz')
+
+    global_arguments = arguments['global']
+    assert global_arguments.overrides == ['foo.bar=baz']
+
+
+def test_parse_arguments_with_multiple_overrides_parses():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--override', 'foo.bar=baz', 'foo.quux=7')
+
+    global_arguments = arguments['global']
+    assert global_arguments.overrides == ['foo.bar=baz', 'foo.quux=7']
+
+
+def test_parse_arguments_with_multiple_overrides_and_flags_parses():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments(
+        '--override', 'foo.bar=baz', '--override', 'foo.quux=7', 'this.that=8'
+    )
+
+    global_arguments = arguments['global']
+    assert global_arguments.overrides == ['foo.bar=baz', 'foo.quux=7', 'this.that=8']
+
+
 def test_parse_arguments_with_list_json_overrides_default():
     arguments = module.parse_arguments('list', '--json')
 

tests/integration/config/test_generate.py

@@ -87,8 +87,8 @@ location:
     assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
 
 
-def test_render_configuration_converts_configuration_to_yaml_string():
-    yaml_string = module._render_configuration({'foo': 'bar'})
+def testrender_configuration_converts_configuration_to_yaml_string():
+    yaml_string = module.render_configuration({'foo': 'bar'})
 
     assert yaml_string == 'foo: bar\n'
 
@@ -194,7 +194,7 @@ def test_generate_sample_configuration_does_not_raise():
     flexmock(module.yaml).should_receive('round_trip_load')
     flexmock(module).should_receive('_schema_to_sample_configuration')
     flexmock(module).should_receive('merge_source_configuration_into_destination')
-    flexmock(module).should_receive('_render_configuration')
+    flexmock(module).should_receive('render_configuration')
     flexmock(module).should_receive('_comment_out_optional_configuration')
     flexmock(module).should_receive('write_configuration')
 
@@ -208,7 +208,7 @@ def test_generate_sample_configuration_with_source_filename_does_not_raise():
     flexmock(module.load).should_receive('load_configuration')
     flexmock(module).should_receive('_schema_to_sample_configuration')
     flexmock(module).should_receive('merge_source_configuration_into_destination')
-    flexmock(module).should_receive('_render_configuration')
+    flexmock(module).should_receive('render_configuration')
     flexmock(module).should_receive('_comment_out_optional_configuration')
     flexmock(module).should_receive('write_configuration')
 

tests/integration/test_execute.py

@@ -88,13 +88,21 @@ def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdou
 
 def test_log_outputs_kills_other_processes_when_one_errors():
     flexmock(module.logger).should_receive('log')
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
     flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, None, 'borg'
+    ).and_return(False)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, 2, 'borg'
+    ).and_return(True)
     other_process = subprocess.Popen(
         ['watch', 'true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
     )
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        other_process, None, 'borg'
+    ).and_return(False)
     flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
         process.stdout
     )
@@ -118,10 +126,15 @@ def test_log_outputs_kills_other_processes_when_one_errors():
 def test_log_outputs_truncates_long_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
     flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, None, 'borg'
+    ).and_return(False)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, 2, 'borg'
+    ).and_return(True)
     flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     with pytest.raises(subprocess.CalledProcessError) as error:

tests/unit/borg/test_create.py

@@ -1326,6 +1326,34 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     )
 
 
+def test_create_archive_with_repository_accepts_borg_placeholders():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('_expand_directories').and_return(())
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '{fqdn}::Documents_{hostname}-{now}', 'foo', 'bar'),
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='{fqdn}',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['{fqdn}'],
+            'exclude_patterns': None,
+        },
+        storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},
+    )
+
+
 def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))

tests/unit/commands/test_borgmatic.py

@@ -447,7 +447,9 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
         ['baz']
     )
-    flexmock(module.sys.stdout).should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    stdout = flexmock()
+    stdout.should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    flexmock(module.sys).stdout = stdout
     arguments = {}
 
     tuple(

tests/unit/hooks/test_dispatch.py

@@ -58,7 +58,7 @@ def test_call_hooks_calls_each_hook_and_collects_return_values():
     assert return_values == expected_return_values
 
 
-def test_call_hooks_calls_skips_return_values_for_unconfigured_hooks():
+def test_call_hooks_calls_skips_return_values_for_missing_hooks():
     hooks = {'super_hook': flexmock()}
     expected_return_values = {'super_hook': flexmock()}
     flexmock(module).should_receive('call_hook').and_return(expected_return_values['super_hook'])
@@ -66,3 +66,13 @@ def test_call_hooks_calls_skips_return_values_for_unconfigured_hooks():
     return_values = module.call_hooks('do_stuff', hooks, 'prefix', ('super_hook', 'other_hook'), 55)
 
     assert return_values == expected_return_values
+
+
+def test_call_hooks_calls_skips_return_values_for_null_hooks():
+    hooks = {'super_hook': flexmock(), 'other_hook': None}
+    expected_return_values = {'super_hook': flexmock()}
+    flexmock(module).should_receive('call_hook').and_return(expected_return_values['super_hook'])
+
+    return_values = module.call_hooks('do_stuff', hooks, 'prefix', ('super_hook', 'other_hook'), 55)
+
+    assert return_values == expected_return_values

tests/unit/hooks/test_healthchecks.py

@@ -28,7 +28,9 @@ def test_forgetful_buffering_handler_emit_forgets_log_records_when_capacity_reac
 def test_format_buffered_logs_for_payload_flattens_log_buffer():
     handler = module.Forgetful_buffering_handler(byte_capacity=100, log_level=1)
     handler.buffer = ['foo\n', 'bar\n']
-    flexmock(module.logging).should_receive('getLogger').and_return(flexmock(handlers=[handler]))
+    logger = flexmock(handlers=[handler])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 
@@ -39,7 +41,9 @@ def test_format_buffered_logs_for_payload_inserts_truncation_indicator_when_logs
     handler = module.Forgetful_buffering_handler(byte_capacity=100, log_level=1)
     handler.buffer = ['foo\n', 'bar\n']
     handler.forgot = True
-    flexmock(module.logging).should_receive('getLogger').and_return(flexmock(handlers=[handler]))
+    logger = flexmock(handlers=[handler])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 
@@ -47,9 +51,9 @@ def test_format_buffered_logs_for_payload_inserts_truncation_indicator_when_logs
 
 
 def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload():
-    flexmock(module.logging).should_receive('getLogger').and_return(
-        flexmock(handlers=[module.logging.Handler()])
-    )
+    logger = flexmock(handlers=[module.logging.Handler()])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35,py36,py37,py38
+envlist = py36,py37,py38,py39
 skip_missing_interpreters = True
 skipsdist = True
 minversion = 3.14.1
@@ -13,8 +13,8 @@ whitelist_externals =
 passenv = COVERAGE_FILE
 commands =
     pytest {posargs}
-    py36,py37,py38: black --check .
-    isort --recursive --check-only --settings-path setup.cfg .
+    py38,py39: black --check .
+    isort --check-only --settings-path setup.cfg .
     flake8 borgmatic tests
 
 [testenv:black]
@@ -34,4 +34,4 @@ commands =
 [testenv:isort]
 deps = {[testenv]deps}
 commands =
-    isort {posargs:--recursive} --settings-path setup.cfg .
+    isort --settings-path setup.cfg .