Fix traceback that can occur when dumping a database (#440).

tests/integration/test_execute: use plain Python rather than xxd

.drone.yml

@@ -1,10 +1,10 @@
 ---
 kind: pipeline
-name: python-3-5-alpine-3-10
+name: python-3-6-alpine-3-9
 
 services:
   - name: postgresql
-    image: postgres:11.6-alpine
+    image: postgres:11.9-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -16,29 +16,7 @@ services:
 
 steps:
 - name: build
-  image: python:3.5-alpine3.10
-  pull: always
-  commands:
-    - scripts/run-full-tests
----
-kind: pipeline
-name: python-3-6-alpine-3-10
-
-services:
-  - name: postgresql
-    image: postgres:11.6-alpine
-    environment:
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: test
-  - name: mysql
-    image: mariadb:10.3
-    environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-
-steps:
-- name: build
-  image: python:3.6-alpine3.10
+  image: alpine:3.9
   pull: always
   commands:
     - scripts/run-full-tests
@@ -48,7 +26,7 @@ name: python-3-7-alpine-3-10
 
 services:
   - name: postgresql
-    image: postgres:11.6-alpine
+    image: postgres:11.9-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -60,51 +38,29 @@ services:
 
 steps:
 - name: build
-  image: python:3.7-alpine3.10
+  image: alpine:3.10
   pull: always
   commands:
     - scripts/run-full-tests
 ---
 kind: pipeline
-name: python-3-7-alpine-3-7
+name: python-3-8-alpine-3-13
 
 services:
   - name: postgresql
-    image: postgres:10.11-alpine
+    image: postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   - name: mysql
-    image: mariadb:10.1
+    image: mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
 
 steps:
 - name: build
-  image: python:3.7-alpine3.7
-  pull: always
-  commands:
-    - scripts/run-full-tests
----
-kind: pipeline
-name: python-3-8-alpine-3-10
-
-services:
-  - name: postgresql
-    image: postgres:11.6-alpine
-    environment:
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: test
-  - name: mysql
-    image: mariadb:10.3
-    environment:
-      MYSQL_ROOT_PASSWORD: test
-      MYSQL_DATABASE: test
-
-steps:
-- name: build
-  image: python:3.8-alpine3.10
+  image: alpine:3.13
   pull: always
   commands:
     - scripts/run-full-tests
@@ -114,7 +70,9 @@ name: documentation
 
 steps:
 - name: build
-  image: plugins/docker
+  #image: plugins/docker
+  # Temporary work-around for https://github.com/drone-plugins/drone-docker/pull/327
+  image: techknowlogick/drone-docker
   settings:
     username:
       from_secret: docker_username
@@ -122,6 +80,7 @@ steps:
       from_secret: docker_password
     repo: witten/borgmatic-docs
     dockerfile: docs/Dockerfile
-  when:
-    branch:
-      - master
+
+trigger:
+  branch:
+    - master

.gitignore

@@ -2,7 +2,7 @@
 *.pyc
 *.swp
 .cache
-.coverage
+.coverage*
 .pytest_cache
 .tox
 __pycache__

NEWS

@@ -1,3 +1,48 @@
+1.5.18
+ * #389: Fix "message too long" error when logging to rsyslog.
+ * #440: Fix traceback that can occur when dumping a database.
+
+1.5.17
+ * #437: Fix error when configuration file contains "umask" option.
+ * Remove test dependency on vim and /dev/urandom.
+
+1.5.16
+ * #379: Suppress console output in sample crontab and systemd service files.
+ * #407: Fix syslog logging on FreeBSD.
+ * #430: Fix hang when restoring a PostgreSQL "tar" format database dump.
+ * Better error messages! Switch the library used for validating configuration files (from pykwalify
+   to jsonschema).
+ * Link borgmatic Ansible role from installation documentation:
+   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install
+
+1.5.15
+ * #419: Document use case of running backups conditionally based on laptop power level:
+   https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
+ * #425: Run arbitrary Borg commands with new "borgmatic borg" action. See the documentation for
+   more information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/
+
+1.5.14
+ * #390: Add link to Hetzner storage offering from the documentation.
+ * #398: Clarify canonical home of borgmatic in documentation.
+ * #406: Clarify that spaces in path names should not be backslashed in path names.
+ * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory!
+ * Fix build so as not to attempt to build and push documentation for a non-master branch.
+ * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13.
+ * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama.
+   IRC connection info: https://torsion.org/borgmatic/#issues
+
+1.5.13
+ * #373: Document that passphrase is used for Borg keyfile encryption, not just repokey encryption.
+ * #404: Add support for ruamel.yaml 0.17.x YAML parsing library.
+ * Update systemd service example to return a permission error when a system call isn't permitted
+   (instead of terminating borgmatic outright).
+ * Drop support for Python 3.5, which has been end-of-lifed.
+ * Add support for Python 3.9.
+ * Update versions of test dependencies (test_requirements.txt and test containers).
+ * Only support black code formatter on Python 3.8+. New black dependencies make installation
+   difficult on older versions of Python.
+ * Replace "improve this documentation" form with link to support and ticket tracker.
+
 1.5.12
  * Fix for previous release with incorrect version suffix in setup.py. No other changes.
 

README.md

@@ -11,6 +11,8 @@ borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
 databases too. Monitor it all with integrated third-party services.
 
+The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
+
 Here's an example configuration file:
 
 ```yaml
@@ -84,10 +86,10 @@ reference guides</a>.
 
 ## Hosting providers
 
-Need somewhere to store your encrypted offsite backups? The following hosting
-providers include specific support for Borg/borgmatic. Using these links and
-services helps support borgmatic development and hosting. (These are referral
-links, but without any tracking scripts or cookies.)
+Need somewhere to store your encrypted off-site backups? The following hosting
+providers include specific support for Borg/borgmatic—and fund borgmatic
+development and hosting when you use these links to sign up. (These are
+referral links, but without any tracking scripts or cookies.)
 
 <ul>
  <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
@@ -95,6 +97,10 @@ links, but without any tracking scripts or cookies.)
  <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
+Additionally, [Hetzner](https://www.hetzner.com/storage/storage-box) has a
+compatible storage offering, but does not currently fund borgmatic
+development or hosting.
+
 ## Support and contributing
 
 ### Issues
@@ -106,9 +112,11 @@ first](https://projects.torsion.org/user/login). Note that you can login with
 an existing GitHub account if you prefer.
 
 If you'd like to chat with borgmatic developers or users, head on over to the
-`#borgmatic` IRC channel on Freenode, either via <a
-href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
-native <a href="irc://chat.freenode.net:6697">IRC client</a>.
+`#borgmatic` IRC channel on Libera Chat, either via <a
+href="https://web.libera.chat/#borgmatic">web chat</a> or a
+native <a href="ircs://irc.libera.chat:6697">IRC client</a>. If you
+don't get a response right away, please hang around a while—or file a ticket
+instead.
 
 Also see the [security
 policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
@@ -120,9 +128,9 @@ Other questions or comments? Contact
 
 ### Contributing
 
-borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://projects.torsion.org/witten/borgmatic), and is also
-mirrored on [GitHub](https://github.com/witten/borgmatic) for convenience.
+borgmatic [source code is
+available](https://projects.torsion.org/witten/borgmatic) and is also mirrored
+on [GitHub](https://github.com/witten/borgmatic) for convenience.
 
 borgmatic is licensed under the GNU General Public License version 3 or any
 later version.

SECURITY.md

@@ -6,14 +6,13 @@ permalink: security-policy/index.html
 ## Supported versions
 
 While we want to hear about security vulnerabilities in all versions of
-borgmatic, security fixes will only be made to the most recently released
-version. It's not practical for our small volunteer effort to maintain
-multiple different release branches and put out separate security patches for
-each.
+borgmatic, security fixes are only made to the most recently released version.
+It's simply not practical for our small volunteer effort to maintain multiple
+release branches and put out separate security patches for each.
 
 ## Reporting a vulnerability
 
 If you find a security vulnerability, please [file a
 ticket](https://torsion.org/borgmatic/#issues) or [send email
 directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
-back within a few days at most, and generally sooner.
+back within a few days at most and generally sooner.

borgmatic/borg/borg.py

@@ -0,0 +1,45 @@
+import logging
+
+from borgmatic.borg.flags import make_flags
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+REPOSITORYLESS_BORG_COMMANDS = {'serve', None}
+
+
+def run_arbitrary_borg(
+    repository, storage_config, options, archive=None, local_path='borg', remote_path=None
+):
+    '''
+    Given a local or remote repository path, a storage config dict, a sequence of arbitrary
+    command-line Borg options, and an optional archive name, run an arbitrary Borg command on the
+    given repository/archive.
+    '''
+    lock_wait = storage_config.get('lock_wait', None)
+
+    try:
+        options = options[1:] if options[0] == '--' else options
+        borg_command = options[0]
+        command_options = tuple(options[1:])
+    except IndexError:
+        borg_command = None
+        command_options = ()
+
+    repository_archive = '::'.join((repository, archive)) if repository and archive else repository
+
+    full_command = (
+        (local_path,)
+        + ((borg_command,) if borg_command else ())
+        + ((repository_archive,) if borg_command and repository_archive else ())
+        + command_options
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+    )
+
+    return execute_command(
+        full_command, output_log_level=logging.WARNING, borg_local_path=local_path,
+    )

borgmatic/borg/create.py

@@ -220,7 +220,8 @@ def create_archive(
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
 
     full_command = (
-        (local_path, 'create')
+        tuple(local_path.split(' '))
+        + ('create',)
         + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
         + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())

borgmatic/commands/arguments.py

@@ -15,17 +15,18 @@ SUBPARSER_ALIASES = {
     'restore': ['--restore', '-r'],
     'list': ['--list', '-l'],
     'info': ['--info', '-i'],
+    'borg': [],
 }
 
 
 def parse_subparser_arguments(unparsed_arguments, subparsers):
     '''
-    Given a sequence of arguments, and a subparsers object as returned by
-    argparse.ArgumentParser().add_subparsers(), give each requested action's subparser a shot at
-    parsing all arguments. This allows common arguments like "--repository" to be shared across
-    multiple subparsers.
+    Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser
+    instance, give each requested action's subparser a shot at parsing all arguments. This allows
+    common arguments like "--repository" to be shared across multiple subparsers.
 
-    Return the result as a dict mapping from subparser name to a parsed namespace of arguments.
+    Return the result as a tuple of (a dict mapping from subparser name to a parsed namespace of
+    arguments, a list of remaining arguments not claimed by any subparser).
     '''
     arguments = collections.OrderedDict()
     remaining_arguments = list(unparsed_arguments)
@@ -35,7 +36,12 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
         for alias in aliases
     }
 
-    for subparser_name, subparser in subparsers.choices.items():
+    # If the "borg" action is used, skip all other subparsers. This avoids confusion like
+    # "borg list" triggering borgmatic's own list action.
+    if 'borg' in unparsed_arguments:
+        subparsers = {'borg': subparsers['borg']}
+
+    for subparser_name, subparser in subparsers.items():
         if subparser_name not in remaining_arguments:
             continue
 
@@ -47,11 +53,11 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
         parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
         for value in vars(parsed).values():
             if isinstance(value, str):
-                if value in subparsers.choices:
+                if value in subparsers:
                     remaining_arguments.remove(value)
             elif isinstance(value, list):
                 for item in value:
-                    if item in subparsers.choices:
+                    if item in subparsers:
                         remaining_arguments.remove(item)
 
         arguments[canonical_name] = parsed
@@ -59,47 +65,33 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
     # If no actions are explicitly requested, assume defaults: prune, create, and check.
     if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
         for subparser_name in ('prune', 'create', 'check'):
-            subparser = subparsers.choices[subparser_name]
+            subparser = subparsers[subparser_name]
             parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
             arguments[subparser_name] = parsed
 
-    return arguments
-
-
-def parse_global_arguments(unparsed_arguments, top_level_parser, subparsers):
-    '''
-    Given a sequence of arguments, a top-level parser (containing subparsers), and a subparsers
-    object as returned by argparse.ArgumentParser().add_subparsers(), parse and return any global
-    arguments as a parsed argparse.Namespace instance.
-    '''
-    # Ask each subparser, one by one, to greedily consume arguments. Any arguments that remain
-    # are global arguments.
     remaining_arguments = list(unparsed_arguments)
-    present_subparser_names = set()
 
-    for subparser_name, subparser in subparsers.choices.items():
-        if subparser_name not in remaining_arguments:
+    # Now ask each subparser, one by one, to greedily consume arguments.
+    for subparser_name, subparser in subparsers.items():
+        if subparser_name not in arguments.keys():
             continue
 
-        present_subparser_names.add(subparser_name)
+        subparser = subparsers[subparser_name]
         unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
 
-    # If no actions are explicitly requested, assume defaults: prune, create, and check.
-    if (
-        not present_subparser_names
-        and '--help' not in unparsed_arguments
-        and '-h' not in unparsed_arguments
-    ):
-        for subparser_name in ('prune', 'create', 'check'):
-            subparser = subparsers.choices[subparser_name]
-            unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+    # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the
+    # "borg" action.
+    if 'borg' in arguments:
+        borg_options_index = remaining_arguments.index('borg') + 1
+        arguments['borg'].options = remaining_arguments[borg_options_index:]
+        remaining_arguments = remaining_arguments[:borg_options_index]
 
     # Remove the subparser names themselves.
-    for subparser_name in present_subparser_names:
+    for subparser_name, subparser in subparsers.items():
         if subparser_name in remaining_arguments:
             remaining_arguments.remove(subparser_name)
 
-    return top_level_parser.parse_args(remaining_arguments)
+    return (arguments, remaining_arguments)
 
 
 class Extend_action(Action):
@@ -510,8 +502,7 @@ def parse_arguments(*unparsed_arguments):
     )
     list_group = list_parser.add_argument_group('list arguments')
     list_group.add_argument(
-        '--repository',
-        help='Path of repository to list, defaults to the configured repository if there is only one',
+        '--repository', help='Path of repository to list, defaults to the configured repositories',
     )
     list_group.add_argument('--archive', help='Name of archive to list (or "latest")')
     list_group.add_argument(
@@ -601,8 +592,32 @@ def parse_arguments(*unparsed_arguments):
     )
     info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    arguments = parse_subparser_arguments(unparsed_arguments, subparsers)
-    arguments['global'] = parse_global_arguments(unparsed_arguments, top_level_parser, subparsers)
+    borg_parser = subparsers.add_parser(
+        'borg',
+        aliases=SUBPARSER_ALIASES['borg'],
+        help='Run an arbitrary Borg command',
+        description='Run an arbitrary Borg command based on borgmatic\'s configuration',
+        add_help=False,
+    )
+    borg_group = borg_parser.add_argument_group('borg arguments')
+    borg_group.add_argument(
+        '--repository',
+        help='Path of repository to pass to Borg, defaults to the configured repositories',
+    )
+    borg_group.add_argument('--archive', help='Name of archive to pass to Borg (or "latest")')
+    borg_group.add_argument(
+        '--',
+        metavar='OPTION',
+        dest='options',
+        nargs='+',
+        help='Options to pass to Borg, command first ("create", "list", etc). "--" is optional. To specify the repository or the archive, you must use --repository or --archive instead of providing them here.',
+    )
+    borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    arguments, remaining_arguments = parse_subparser_arguments(
+        unparsed_arguments, subparsers.choices
+    )
+    arguments['global'] = top_level_parser.parse_args(remaining_arguments)
 
     if arguments['global'].excludes_filename:
         raise ValueError(

borgmatic/commands/borgmatic.py

@@ -9,6 +9,7 @@ from subprocess import CalledProcessError
 import colorama
 import pkg_resources
 
+from borgmatic.borg import borg as borg_borg
 from borgmatic.borg import check as borg_check
 from borgmatic.borg import create as borg_create
 from borgmatic.borg import environment as borg_environment
@@ -543,6 +544,22 @@ def run_actions(
             )
             if json_output:
                 yield json.loads(json_output)
+    if 'borg' in arguments:
+        if arguments['borg'].repository is None or validate.repositories_match(
+            repository, arguments['borg'].repository
+        ):
+            logger.warning('{}: Running arbitrary Borg command'.format(repository))
+            archive_name = borg_list.resolve_archive_name(
+                repository, arguments['borg'].archive, storage, local_path, remote_path
+            )
+            borg_borg.run_arbitrary_borg(
+                repository,
+                storage,
+                options=arguments['borg'].options,
+                archive=archive_name,
+                local_path=local_path,
+                remote_path=remote_path,
+            )
 
 
 def load_configurations(config_filenames, overrides=None):

borgmatic/config/convert.py

@@ -17,7 +17,7 @@ def _convert_section(source_section_config, section_schema):
             (
                 option_name,
                 int(option_value)
-                if section_schema['map'].get(option_name, {}).get('type') == 'int'
+                if section_schema['properties'].get(option_name, {}).get('type') == 'integer'
                 else option_value,
             )
             for option_name, option_value in source_section_config.items()
@@ -38,7 +38,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
     '''
     destination_config = yaml.comments.CommentedMap(
         [
-            (section_name, _convert_section(section_config, schema['map'][section_name]))
+            (section_name, _convert_section(section_config, schema['properties'][section_name]))
             for section_name, section_config in source_config._asdict().items()
         ]
     )
@@ -54,11 +54,11 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
     # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration_map(destination_config, schema)
+    generate.add_comments_to_configuration_object(destination_config, schema)
 
     for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration_map(
-            section_config, schema['map'][section_name], indent=generate.INDENT
+        generate.add_comments_to_configuration_object(
+            section_config, schema['properties'][section_name], indent=generate.INDENT
         )
 
     return destination_config

borgmatic/config/generate.py

@@ -24,29 +24,27 @@ def _insert_newline_before_comment(config, field_name):
 def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema, generate and return sample config for it. Include comments
-    for each section based on the schema "desc" description.
+    for each section based on the schema "description".
     '''
+    schema_type = schema.get('type')
     example = schema.get('example')
     if example is not None:
         return example
 
-    if 'seq' in schema:
+    if schema_type == 'array':
         config = yaml.comments.CommentedSeq(
-            [
-                _schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
-                for item_schema in schema['seq']
-            ]
+            [_schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
         )
         add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
-    elif 'map' in schema:
+    elif schema_type == 'object':
         config = yaml.comments.CommentedMap(
             [
                 (field_name, _schema_to_sample_configuration(sub_schema, level + 1))
-                for field_name, sub_schema in schema['map'].items()
+                for field_name, sub_schema in schema['properties'].items()
             ]
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
-        add_comments_to_configuration_map(
+        add_comments_to_configuration_object(
             config, schema, indent=indent, skip_first=parent_is_sequence
         )
     else:
@@ -132,8 +130,8 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
 
 def add_comments_to_configuration_sequence(config, schema, indent=0):
     '''
-    If the given config sequence's items are maps, then mine the schema for the description of the
-    map's first item, and slap that atop the sequence. Indent the comment the given number of
+    If the given config sequence's items are object, then mine the schema for the description of the
+    object's first item, and slap that atop the sequence. Indent the comment the given number of
     characters.
 
     Doing this for sequences of maps results in nice comments that look like:
@@ -142,16 +140,16 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
     things:
         # First key description. Added by this function.
         - key: foo
-          # Second key description. Added by add_comments_to_configuration_map().
+          # Second key description. Added by add_comments_to_configuration_object().
           other: bar
     ```
     '''
-    if 'map' not in schema['seq'][0]:
+    if schema['items'].get('type') != 'object':
         return
 
     for field_name in config[0].keys():
-        field_schema = schema['seq'][0]['map'].get(field_name, {})
-        description = field_schema.get('desc')
+        field_schema = schema['items']['properties'].get(field_name, {})
+        description = field_schema.get('description')
 
         # No description to use? Skip it.
         if not field_schema or not description:
@@ -160,7 +158,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
         config[0].yaml_set_start_comment(description, indent=indent)
 
         # We only want the first key's description here, as the rest of the keys get commented by
-        # add_comments_to_configuration_map().
+        # add_comments_to_configuration_object().
         return
 
 
@@ -169,7 +167,7 @@ REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
 COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
 
 
-def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
+def add_comments_to_configuration_object(config, schema, indent=0, skip_first=False):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
     config mapping, before each field. Indent the comment the given number of characters.
@@ -178,8 +176,8 @@ def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False
         if skip_first and index == 0:
             continue
 
-        field_schema = schema['map'].get(field_name, {})
-        description = field_schema.get('desc', '').strip()
+        field_schema = schema['properties'].get(field_name, {})
+        description = field_schema.get('description', '').strip()
 
         # If this is an optional key, add an indicator to the comment flagging it to be commented
         # out from the sample configuration. This sentinel is consumed by downstream processing that
@@ -268,9 +266,9 @@ def merge_source_configuration_into_destination(destination_config, source_confi
 def generate_sample_configuration(source_filename, destination_filename, schema_filename):
     '''
     Given an optional source configuration filename, and a required destination configuration
-    filename, and the path to a schema filename in pykwalify YAML schema format, write out a
-    sample configuration file based on that schema. If a source filename is provided, merge the
-    parsed contents of that configuration into the generated configuration.
+    filename, and the path to a schema filename in a YAML rendition of the JSON Schema format,
+    write out a sample configuration file based on that schema. If a source filename is provided,
+    merge the parsed contents of that configuration into the generated configuration.
     '''
     schema = yaml.round_trip_load(open(schema_filename))
     source_config = None

borgmatic/config/schema.yaml

@@ -1,30 +1,37 @@
-name: Borgmatic configuration file schema
-version: 1
-map:
+type: object
+required:
+    - location
+additionalProperties: false
+properties:
     location:
-        desc: |
+        type: object
+        description: |
             Where to look for files to backup, and where to store those backups.
             See https://borgbackup.readthedocs.io/en/stable/quickstart.html and
             https://borgbackup.readthedocs.io/en/stable/usage/create.html
             for details.
-        required: true
-        map:
+        required:
+            - source_directories
+            - repositories
+        additionalProperties: false
+        properties:
             source_directories:
-                required: true
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of source directories to backup (required). Globs and
-                    tildes are expanded.
+                    tildes are expanded. Do not backslash spaces in path names.
                 example:
                     - /home
                     - /etc
                     - /var/log/syslog*
+                    - /home/user/path with spaces
             repositories:
-                required: true
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     Paths to local or remote repositories (required). Tildes are
                     expanded. Multiple repositories are backed up to in
                     sequence. Borg placeholders can be used. See the output of
@@ -36,36 +43,36 @@ map:
                     - user@backupserver:sourcehostname.borg
                     - "user@backupserver:{fqdn}"
             one_file_system:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Stay in same file system (do not cross mount points).
                     Defaults to false. But when a database hook is used, the
                     setting here is ignored and one_file_system is considered
                     true.
                 example: true
             numeric_owner:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Only store/extract numeric user and group identifiers.
                     Defaults to false.
                 example: true
             atime:
-                type: bool
-                desc: Store atime into archive. Defaults to true.
+                type: boolean
+                description: Store atime into archive. Defaults to true.
                 example: false
             ctime:
-                type: bool
-                desc: Store ctime into archive. Defaults to true.
+                type: boolean
+                description: Store ctime into archive. Defaults to true.
                 example: false
             birthtime:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Store birthtime (creation date) into archive. Defaults to
                     true.
                 example: false
             read_special:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Use Borg's --read-special flag to allow backup of block and
                     other special devices. Use with caution, as it will lead to
                     problems if used when backing up special devices such as
@@ -74,30 +81,33 @@ map:
                     considered true.
                 example: false
             bsd_flags:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive.
                     Defaults to true.
                 example: true
             files_cache:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Mode in which to operate the files cache. See
                     http://borgbackup.readthedocs.io/en/stable/usage/create.html
                     for details. Defaults to "ctime,size,inode".
                 example: ctime,size,inode
             local_path:
-                type: str
-                desc: Alternate Borg local executable. Defaults to "borg".
+                type: string
+                description: |
+                    Alternate Borg local executable. Defaults to "borg".
                 example: borg1
             remote_path:
-                type: str
-                desc: Alternate Borg remote executable. Defaults to "borg".
+                type: string
+                description: |
+                    Alternate Borg remote executable. Defaults to "borg".
                 example: borg1
             patterns:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     Any paths matching these patterns are included/excluded from
                     backups. Globs are expanded. (Tildes are not.) Note that
                     Borg considers this option experimental. See the output of
@@ -109,9 +119,10 @@ map:
                     - '+ /home/susan'
                     - '- /home/*'
             patterns_from:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     Read include/exclude patterns from one or more separate
                     named files, one pattern per line. Note that Borg considers
                     this option experimental. See the output of "borg help
@@ -119,89 +130,97 @@ map:
                 example:
                     - /etc/borgmatic/patterns
             exclude_patterns:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     Any paths matching these patterns are excluded from backups.
-                    Globs and tildes are expanded. See the output of "borg help
-                    patterns" for more details.
+                    Globs and tildes are expanded. Do not backslash spaces in
+                    path names. See the output of "borg help patterns" for more
+                    details.
                 example:
                     - '*.pyc'
                     - /home/*/.cache
                     - /etc/ssl
+                    - /home/user/path with spaces
             exclude_from:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     Read exclude patterns from one or more separate named files,
                     one pattern per line. See the output of "borg help patterns"
                     for more details.
                 example:
                     - /etc/borgmatic/excludes
             exclude_caches:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Exclude directories that contain a CACHEDIR.TAG file. See
                     http://www.brynosaurus.com/cachedir/spec.html for details.
                     Defaults to false.
                 example: true
             exclude_if_present:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     Exclude directories that contain a file with the given
                     filenames. Defaults to not set.
                 example:
                     - .nobackup
             keep_exclude_tags:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     If true, the exclude_if_present filename is included in
                     backups. Defaults to false, meaning that the
                     exclude_if_present filename is omitted from backups.
                 example: true
             exclude_nodump:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Exclude files with the NODUMP flag. Defaults to false.
                 example: true
             borgmatic_source_directory:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Path for additional source files used for temporary internal
                     state like borgmatic database dumps. Note that changing this
                     path prevents "borgmatic restore" from finding any database
                     dumps created before the change. Defaults to ~/.borgmatic
                 example: /tmp/borgmatic
     storage:
-        desc: |
+        type: object
+        description: |
             Repository storage options. See
             https://borgbackup.readthedocs.io/en/stable/usage/create.html and
             https://borgbackup.readthedocs.io/en/stable/usage/general.html for
             details.
-        map:
+        additionalProperties: false
+        properties:
             encryption_passcommand:
-                type: str
-                desc: |
+                type: string
+                description: |
                     The standard output of this command is used to unlock the
                     encryption key. Only use on repositories that were
-                    initialized with passcommand/repokey encryption. Note that
-                    if both encryption_passcommand and encryption_passphrase are
-                    set, then encryption_passphrase takes precedence. Defaults
-                    to not set.
+                    initialized with passcommand/repokey/keyfile encryption.
+                    Note that if both encryption_passcommand and
+                    encryption_passphrase are set, then encryption_passphrase
+                    takes precedence. Defaults to not set.
                 example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Passphrase to unlock the encryption key with. Only use on
-                    repositories that were initialized with passphrase/repokey
-                    encryption. Quote the value if it contains punctuation, so
-                    it parses correctly. And backslash any quote or backslash
-                    literals as well. Defaults to not set.
+                    repositories that were initialized with
+                    passphrase/repokey/keyfile encryption. Quote the value if it
+                    contains punctuation, so it parses correctly. And backslash
+                    any quote or backslash literals as well. Defaults to not
+                    set.
                 example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
             checkpoint_interval:
-                type: int
-                desc: |
+                type: integer
+                description: |
                     Number of seconds between each checkpoint during a
                     long-running backup. See
                     https://borgbackup.readthedocs.io/en/stable/faq.html
@@ -209,8 +228,8 @@ map:
                     minutes).
                 example: 1800
             chunker_params:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Specify the parameters passed to then chunker
                     (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS,
                     HASH_WINDOW_SIZE). See
@@ -218,73 +237,73 @@ map:
                     for details. Defaults to "19,23,21,4095".
                 example: 19,23,21,4095
             compression:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Type of compression to use when creating archives. See
                     http://borgbackup.readthedocs.io/en/stable/usage/create.html
                     for details. Defaults to "lz4".
                 example: lz4
             remote_rate_limit:
-                type: int
-                desc: |
+                type: integer
+                description: |
                     Remote network upload rate limit in kiBytes/second. Defaults
                     to unlimited.
                 example: 100
             temporary_directory:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Directory where temporary files are stored. Defaults to
                     $TMPDIR
                 example: /path/to/tmpdir
             ssh_command:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Command to use instead of "ssh". This can be used to specify
                     ssh options.  Defaults to not set.
                 example: ssh -i /path/to/private/key
             borg_base_directory:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Base path used for various Borg directories. Defaults to
                     $HOME, ~$USER, or ~.
                 example: /path/to/base
             borg_config_directory:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Path for Borg configuration files. Defaults to
                     $borg_base_directory/.config/borg
                 example: /path/to/base/config
             borg_cache_directory:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Path for Borg cache files. Defaults to
                     $borg_base_directory/.cache/borg
                 example: /path/to/base/cache
             borg_security_directory:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Path for Borg security and encryption nonce files. Defaults
                     to $borg_base_directory/.config/borg/security
                 example: /path/to/base/config/security
             borg_keys_directory:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Path for Borg encryption key files. Defaults to
                     $borg_base_directory/.config/borg/keys
                 example: /path/to/base/config/keys
             umask:
-                type: scalar
-                desc: Umask to be used for borg create. Defaults to 0077.
+                type: integer
+                description: Umask to be used for borg create. Defaults to 0077.
                 example: 0077
             lock_wait:
-                type: int
-                desc: |
+                type: integer
+                description: |
                     Maximum seconds to wait for acquiring a repository/cache
                     lock. Defaults to 1.
                 example: 5
             archive_name_format:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Name of the archive. Borg placeholders can be used. See the
                     output of "borg help placeholders" for details. Defaults to
                     "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this
@@ -294,40 +313,42 @@ map:
                     prefix in the consistency section as well.
                 example: "{hostname}-documents-{now}"
             relocated_repo_access_is_ok:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Bypass Borg error about a repository that has been moved.
                     Defaults to false.
                 example: true
             unknown_unencrypted_repo_access_is_ok:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Bypass Borg error about a previously unknown unencrypted
                     repository. Defaults to false.
                 example: true
             extra_borg_options:
-                map:
+                type: object
+                additionalProperties: false
+                properties:
                     init:
-                        type: str
-                        desc: |
+                        type: string
+                        description: |
                             Extra command-line options to pass to "borg init".
                         example: "--make-parent-dirs"
                     prune:
-                        type: str
-                        desc: |
+                        type: string
+                        description: |
                             Extra command-line options to pass to "borg prune".
                         example: "--save-space"
                     create:
-                        type: str
-                        desc: |
+                        type: string
+                        description: |
                             Extra command-line options to pass to "borg create".
                         example: "--no-files-cache"
                     check:
-                        type: str
-                        desc: |
+                        type: string
+                        description: |
                             Extra command-line options to pass to "borg check".
                         example: "--save-space"
-                desc: |
+                description: |
                     Additional options to pass directly to particular Borg
                     commands, handy for Borg options that borgmatic does not yet
                     support natively. Note that borgmatic does not perform any
@@ -335,72 +356,76 @@ map:
                     "--verbosity 2" shows the exact Borg command-line
                     invocation.
     retention:
-        desc: |
+        type: object
+        description: |
             Retention policy for how many backups to keep in each category. See
             https://borgbackup.readthedocs.io/en/stable/usage/prune.html for
             details. At least one of the "keep" options is required for pruning
             to work. To skip pruning entirely, run "borgmatic create" or "check"
             without the "prune" action. See borgmatic documentation for details.
-        map:
+        additionalProperties: false
+        properties:
             keep_within:
-                type: str
-                desc: Keep all archives within this time interval.
+                type: string
+                description: Keep all archives within this time interval.
                 example: 3H
             keep_secondly:
-                type: int
-                desc: Number of secondly archives to keep.
+                type: integer
+                description: Number of secondly archives to keep.
                 example: 60
             keep_minutely:
-                type: int
-                desc: Number of minutely archives to keep.
+                type: integer
+                description: Number of minutely archives to keep.
                 example: 60
             keep_hourly:
-                type: int
-                desc: Number of hourly archives to keep.
+                type: integer
+                description: Number of hourly archives to keep.
                 example: 24
             keep_daily:
-                type: int
-                desc: Number of daily archives to keep.
+                type: integer
+                description: Number of daily archives to keep.
                 example: 7
             keep_weekly:
-                type: int
-                desc: Number of weekly archives to keep.
+                type: integer
+                description: Number of weekly archives to keep.
                 example: 4
             keep_monthly:
-                type: int
-                desc: Number of monthly archives to keep.
+                type: integer
+                description: Number of monthly archives to keep.
                 example: 6
             keep_yearly:
-                type: int
-                desc: Number of yearly archives to keep.
+                type: integer
+                description: Number of yearly archives to keep.
                 example: 1
             prefix:
-                type: str
-                desc: |
+                type: string
+                description: |
                     When pruning, only consider archive names starting with this
                     prefix.  Borg placeholders can be used. See the output of
                     "borg help placeholders" for details. Defaults to
                     "{hostname}-". Use an empty value to disable the default.
                 example: sourcehostname
     consistency:
-        desc: |
+        type: object
+        description: |
             Consistency checks to run after backups. See
             https://borgbackup.readthedocs.io/en/stable/usage/check.html and
             https://borgbackup.readthedocs.io/en/stable/usage/extract.html for
             details.
-        map:
+        additionalProperties: false
+        properties:
             checks:
-                seq:
-                    - type: str
-                      enum: [
-                          'repository',
-                          'archives',
-                          'data',
-                          'extract',
-                          'disabled'
-                      ]
-                      unique: true
-                desc: |
+                type: array
+                items:
+                    type: string
+                    enum:
+                        - repository
+                        - archives
+                        - data
+                        - extract
+                        - disabled
+                    uniqueItems: true
+                description: |
                     List of one or more consistency checks to run: "repository",
                     "archives", "data", and/or "extract". Defaults to
                     "repository" and "archives". Set to "disabled" to disable
@@ -413,9 +438,10 @@ map:
                     - repository
                     - archives
             check_repositories:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     Paths to a subset of the repositories in the location
                     section on which to run consistency checks. Handy in case
                     some of your repositories are very large, and so running
@@ -425,15 +451,15 @@ map:
                 example:
                     - user@backupserver:sourcehostname.borg
             check_last:
-                type: int
-                desc: |
+                type: integer
+                description: |
                     Restrict the number of checked archives to the last n.
                     Applies only to the "archives" check. Defaults to checking
                     all archives.
                 example: 3
             prefix:
-                type: str
-                desc: |
+                type: string
+                description: |
                     When performing the "archives" check, only consider archive
                     names starting with this prefix. Borg placeholders can be
                     used. See the output of "borg help placeholders" for
@@ -441,101 +467,115 @@ map:
                     disable the default.
                 example: sourcehostname
     output:
-        desc: |
+        type: object
+        description: |
             Options for customizing borgmatic's own output and logging.
-        map:
+        additionalProperties: false
+        properties:
             color:
-                type: bool
-                desc: |
+                type: boolean
+                description: |
                     Apply color to console output. Can be overridden with
                     --no-color command-line flag. Defaults to true.
                 example: false
     hooks:
-        desc: |
+        type: object
+        description: |
             Shell commands, scripts, or integrations to execute at various
             points during a borgmatic run. IMPORTANT: All provided commands and
             scripts are executed with user permissions of borgmatic. Do not
             forget to set secure permissions on this configuration file (chmod
             0600) as well as on any script called from a hook (chmod 0700) to
             prevent potential shell injection or privilege escalation.
-        map:
+        additionalProperties: false
+        properties:
             before_backup:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     before creating a backup, run once per configuration file.
                 example:
                     - echo "Starting a backup."
             before_prune:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     before pruning, run once per configuration file.
                 example:
                     - echo "Starting pruning."
             before_check:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     before consistency checks, run once per configuration file.
                 example:
                     - echo "Starting checks."
             before_extract:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     before extracting a backup, run once per configuration file.
                 example:
                     - echo "Starting extracting."
             after_backup:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     after creating a backup, run once per configuration file.
                 example:
                     - echo "Finished a backup."
             after_prune:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     after pruning, run once per configuration file.
                 example:
                     - echo "Finished pruning."
             after_check:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     after consistency checks, run once per configuration file.
                 example:
                     - echo "Finished checks."
             after_extract:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     after extracting a backup, run once per configuration file.
                 example:
                     - echo "Finished extracting."
             on_error:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     when an exception occurs during a "prune", "create", or
                     "check" action or an associated before/after hook.
                 example:
                     - echo "Error during prune/create/check."
             before_everything:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     before running all actions (if one of them is "create").
                     These are collected from all configuration files and then
@@ -543,9 +583,10 @@ map:
                 example:
                     - echo "Starting actions."
             after_everything:
-                seq:
-                    - type: str
-                desc: |
+                type: array
+                items:
+                    type: string
+                description: |
                     List of one or more shell commands or scripts to execute
                     after running all actions (if one of them is "create").
                     These are collected from all configuration files and then
@@ -553,12 +594,15 @@ map:
                 example:
                     - echo "Completed actions."
             postgresql_databases:
-                seq:
-                    - map:
+                type: array
+                items:
+                    type: object
+                    required: ['name']
+                    additionalProperties: false
+                    properties:
                         name:
-                            required: true
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Database name (required if using this hook). Or
                                 "all" to dump all databases on the host. Note
                                 that using this database hook implicitly enables
@@ -566,26 +610,26 @@ map:
                                 above) to support dump and restore streaming.
                             example: users
                         hostname:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Database hostname to connect to. Defaults to
                                 connecting via local Unix socket.
                             example: database.example.org
                         port:
-                            type: int
-                            desc: Port to connect to. Defaults to 5432.
+                            type: integer
+                            description: Port to connect to. Defaults to 5432.
                             example: 5433
                         username:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Username with which to connect to the database.
                                 Defaults to the username of the current user.
                                 You probably want to specify the "postgres"
                                 superuser here when the database name is "all".
                             example: dbuser
                         password:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Password with which to connect to the database.
                                 Omitting a password will only work if PostgreSQL
                                 is configured to trust the configured username
@@ -593,9 +637,9 @@ map:
                                 file.
                             example: trustsome1
                         format:
-                            type: str
+                            type: string
                             enum: ['plain', 'custom', 'directory', 'tar']
-                            desc: |
+                            description: |
                                 Database dump output format. One of "plain",
                                 "custom", "directory", or "tar". Defaults to
                                 "custom" (unlike raw pg_dump). See pg_dump
@@ -603,45 +647,45 @@ map:
                                 ignored when the database name is "all".
                             example: directory
                         ssl_mode:
-                            type: str
+                            type: string
                             enum: ['disable', 'allow', 'prefer',
                                   'require', 'verify-ca', 'verify-full']
-                            desc: |
+                            description: |
                                 SSL mode to use to connect to the database
                                 server. One of "disable", "allow", "prefer",
                                 "require", "verify-ca" or "verify-full".
                                 Defaults to "disable".
                             example: require
                         ssl_cert:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Path to a client certificate.
                             example: "/root/.postgresql/postgresql.crt"
                         ssl_key:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Path to a private client key.
                             example: "/root/.postgresql/postgresql.key"
                         ssl_root_cert:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Path to a root certificate containing a list of
                                 trusted certificate authorities.
                             example: "/root/.postgresql/root.crt"
                         ssl_crl:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Path to a certificate revocation list.
                             example: "/root/.postgresql/root.crl"
                         options:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Additional pg_dump/pg_dumpall options to pass
                                 directly to the dump command, without performing
                                 any validation on them. See pg_dump
                                 documentation for details.
                             example: --role=someone
-                desc: |
+                description: |
                     List of one or more PostgreSQL databases to dump before
                     creating a backup, run once per configuration file. The
                     database dumps are added to your source directories at
@@ -651,12 +695,15 @@ map:
                     https://www.postgresql.org/docs/current/libpq-ssl.html for
                     details.
             mysql_databases:
-                seq:
-                    - map:
+                type: array
+                items:
+                    type: object
+                    required: ['name']
+                    additionalProperties: false
+                    properties:
                         name:
-                            required: true
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Database name (required if using this hook). Or
                                 "all" to dump all databases on the host. Note
                                 that using this database hook implicitly enables
@@ -664,38 +711,38 @@ map:
                                 above) to support dump and restore streaming.
                             example: users
                         hostname:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Database hostname to connect to. Defaults to
                                 connecting via local Unix socket.
                             example: database.example.org
                         port:
-                            type: int
-                            desc: Port to connect to. Defaults to 3306.
+                            type: integer
+                            description: Port to connect to. Defaults to 3306.
                             example: 3307
                         username:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Username with which to connect to the database.
                                 Defaults to the username of the current user.
                             example: dbuser
                         password:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Password with which to connect to the database.
                                 Omitting a password will only work if MySQL is
                                 configured to trust the configured username
                                 without a password.
                             example: trustsome1
                         options:
-                            type: str
-                            desc: |
+                            type: string
+                            description: |
                                 Additional mysqldump options to pass directly to
                                 the dump command, without performing any
                                 validation on them. See mysqldump documentation
                                 for details.
                             example: --skip-comments
-                desc: |
+                description: |
                     List of one or more MySQL/MariaDB databases to dump before
                     creating a backup, run once per configuration file. The
                     database dumps are added to your source directories at
@@ -704,8 +751,8 @@ map:
                     https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
                     https://mariadb.com/kb/en/library/mysqldump/ for details.
             healthchecks:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Healthchecks ping URL or UUID to notify when a backup
                     begins, ends, or errors. Create an account at
                     https://healthchecks.io if you'd like to use this service.
@@ -713,8 +760,8 @@ map:
                 example:
                     https://hc-ping.com/your-uuid-here
             cronitor:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Cronitor ping URL to notify when a backup begins, ends, or
                     errors. Create an account at https://cronitor.io if you'd
                     like to use this service. See borgmatic monitoring
@@ -722,8 +769,8 @@ map:
                 example:
                     https://cronitor.link/d3x0c1
             pagerduty:
-                type: str
-                desc: |
+                type: string
+                description: |
                     PagerDuty integration key used to notify PagerDuty when a
                     backup errors. Create an account at
                     https://www.pagerduty.com/ if you'd like to use this
@@ -731,8 +778,8 @@ map:
                 example:
                     a177cad45bd374409f78906a810a3074
             cronhub:
-                type: str
-                desc: |
+                type: string
+                description: |
                     Cronhub ping URL to notify when a backup begins, ends, or
                     errors. Create an account at https://cronhub.io if you'd
                     like to use this service. See borgmatic monitoring
@@ -740,8 +787,8 @@ map:
                 example:
                     https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01
             umask:
-                type: scalar
-                desc: |
+                type: integer
+                description: |
                     Umask used when executing hooks. Defaults to the umask that
                     borgmatic is run with.
                 example: 0077

borgmatic/config/validate.py

@@ -1,9 +1,7 @@
-import logging
 import os
 
+import jsonschema
 import pkg_resources
-import pykwalify.core
-import pykwalify.errors
 import ruamel.yaml
 
 from borgmatic.config import load, normalize, override
@@ -17,15 +15,40 @@ def schema_filename():
     return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
 
 
+def format_error_path_element(path_element):
+    '''
+    Given a path element into a JSON data structure, format it for display as a string.
+    '''
+    if isinstance(path_element, int):
+        return str('[{}]'.format(path_element))
+
+    return str('.{}'.format(path_element))
+
+
+def format_error(error):
+    '''
+    Given an instance of jsonschema.exceptions.ValidationError, format it for display as a string.
+    '''
+    if not error.path:
+        return 'At the top level: {}'.format(error.message)
+
+    formatted_path = ''.join(format_error_path_element(element) for element in error.path)
+    return "At '{}': {}".format(formatted_path.lstrip('.'), error.message)
+
+
 class Validation_error(ValueError):
     '''
-    A collection of error message strings generated when attempting to validate a particular
-    configurartion file.
+    A collection of error messages generated when attempting to validate a particular
+    configuration file.
     '''
 
-    def __init__(self, config_filename, error_messages):
+    def __init__(self, config_filename, errors):
+        '''
+        Given a configuration filename path and a sequence of
+        jsonschema.exceptions.ValidationError instances, create a Validation_error.
+        '''
         self.config_filename = config_filename
-        self.error_messages = error_messages
+        self.errors = errors
 
     def __str__(self):
         '''
@@ -33,7 +56,7 @@ class Validation_error(ValueError):
         '''
         return 'An error occurred while parsing a configuration file at {}:\n'.format(
             self.config_filename
-        ) + '\n'.join(self.error_messages)
+        ) + '\n'.join(format_error(error) for error in self.errors)
 
 
 def apply_logical_validation(config_filename, parsed_configuration):
@@ -65,29 +88,12 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
-def remove_examples(schema):
-    '''
-    pykwalify gets angry if the example field is not a string. So rather than bend to its will,
-    remove all examples from the given schema before passing the schema to pykwalify.
-    '''
-    if 'map' in schema:
-        for item_name, item_schema in schema['map'].items():
-            item_schema.pop('example', None)
-            remove_examples(item_schema)
-    elif 'seq' in schema:
-        for item_schema in schema['seq']:
-            item_schema.pop('example', None)
-            remove_examples(item_schema)
-
-    return schema
-
-
 def parse_configuration(config_filename, schema_filename, overrides=None):
     '''
-    Given the path to a config filename in YAML format, the path to a schema filename in pykwalify
-    YAML schema format, a sequence of configuration file override strings in the form of
-    "section.option=value", return the parsed configuration as a data structure of nested dicts and
-    lists corresponding to the schema. Example return value:
+    Given the path to a config filename in YAML format, the path to a schema filename in a YAML
+    rendition of JSON Schema format, a sequence of configuration file override strings in the form
+    of "section.option=value", return the parsed configuration as a data structure of nested dicts
+    and lists corresponding to the schema. Example return value:
 
        {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
        'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
@@ -95,8 +101,6 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
     Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
     have permissions to read the file, or Validation_error if the config does not match the schema.
     '''
-    logging.getLogger('pykwalify').setLevel(logging.ERROR)
-
     try:
         config = load.load_configuration(config_filename)
         schema = load.load_configuration(schema_filename)
@@ -106,15 +110,15 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
     override.apply_overrides(config, overrides)
     normalize.normalize(config)
 
-    validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
-    parsed_result = validator.validate(raise_exception=False)
+    validator = jsonschema.Draft7Validator(schema)
+    validation_errors = tuple(validator.iter_errors(config))
 
-    if validator.validation_errors:
-        raise Validation_error(config_filename, validator.validation_errors)
+    if validation_errors:
+        raise Validation_error(config_filename, validation_errors)
 
-    apply_logical_validation(config_filename, parsed_result)
+    apply_logical_validation(config_filename, config)
 
-    return parsed_result
+    return config
 
 
 def normalize_repository_path(repository):

borgmatic/execute.py

@@ -23,7 +23,7 @@ def exit_code_indicates_error(process, exit_code, borg_local_path=None):
     command = process.args.split(' ') if isinstance(process.args, str) else process.args
 
     if borg_local_path and command[0] == borg_local_path:
-        return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+        return bool(exit_code < 0 or exit_code >= BORG_ERROR_EXIT_CODE)
 
     return bool(exit_code != 0)
 
@@ -59,11 +59,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
     '''
     # Map from output buffer to sequence of last lines.
     buffer_last_lines = collections.defaultdict(list)
-    output_buffers = [
-        output_buffer_for_process(process, exclude_stdouts)
+    process_for_output_buffer = {
+        output_buffer_for_process(process, exclude_stdouts): process
         for process in processes
         if process.stdout or process.stderr
-    ]
+    }
+    output_buffers = list(process_for_output_buffer.keys())
 
     # Log output for each process until they all exit.
     while True:
@@ -71,8 +72,23 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
             (ready_buffers, _, _) = select.select(output_buffers, [], [])
 
             for ready_buffer in ready_buffers:
+                ready_process = process_for_output_buffer.get(ready_buffer)
+
+                # The "ready" process has exited, but it might be a pipe destination with other
+                # processes (pipe sources) waiting to be read from. So as a measure to prevent
+                # hangs, vent all processes when one exits.
+                if ready_process and ready_process.poll() is not None:
+                    for other_process in processes:
+                        if (
+                            other_process.poll() is None
+                            and other_process.stdout
+                            and other_process.stdout not in output_buffers
+                        ):
+                            # Add the process's output to output_buffers to ensure it'll get read.
+                            output_buffers.append(other_process.stdout)
+
                 line = ready_buffer.readline().rstrip().decode()
-                if not line:
+                if not line or not ready_process:
                     continue
 
                 # Keep the last few lines of output in case the process errors, and we need the output for
@@ -123,9 +139,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         if not output_buffer:
             continue
 
-        remaining_output = output_buffer.read().rstrip().decode()
+        while True:  # pragma: no cover
+            remaining_output = output_buffer.readline().rstrip().decode()
+
+            if not remaining_output:
+                break
 
-        if remaining_output:  # pragma: no cover
             logger.log(output_log_level, remaining_output)
 
 

borgmatic/logger.py

@@ -1,4 +1,5 @@
 import logging
+import logging.handlers
 import os
 import sys
 
@@ -151,6 +152,8 @@ def configure_logging(
             syslog_path = '/dev/log'
         elif os.path.exists('/var/run/syslog'):
             syslog_path = '/var/run/syslog'
+        elif os.path.exists('/var/run/log'):
+            syslog_path = '/var/run/log'
 
     if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)

docs/Dockerfile

@@ -1,13 +1,13 @@
-FROM python:3.8.1-alpine3.11 as borgmatic
+FROM python:3.8-alpine3.12 as borgmatic
 
 COPY . /app
-RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
+RUN pip install --no-cache ruamel.yaml.clib==0.2.2 /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in init prune create check extract mount umount restore list info; do \
+    && for action in init prune create check extract export-tar mount umount restore list info borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:13.7.0-alpine as html
+FROM node:15.2.1-alpine as html
 
 ARG ENVIRONMENT=production
 
@@ -26,7 +26,7 @@ COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM nginx:1.16.1-alpine
+FROM nginx:1.19.4-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/_includes/components/suggestion-form.css

@@ -1,18 +0,0 @@
-#suggestion-form textarea {
-	font-family: sans-serif;
-	width: 100%;
-}
-
-#suggestion-form label {
-	font-weight: bold;
-}
-
-#suggestion-form input[type=email] {
-	font-size: 16px;
-	width: 100%;
-}
-
-#suggestion-form .form-error {
-        color: red;
-}
-

docs/_includes/components/suggestion-form.html

@@ -1,33 +0,0 @@
-<h2>Improve this documentation</h2>
-
-<p>Have an idea on how to make this documentation even better? Send your
-feedback below! But if you need help with borgmatic, or have an idea for a
-borgmatic feature, please use our <a href="https://torsion.org/borgmatic/#issues">issue
-tracker</a> instead.</p>
-
-<form id="suggestion-form">
-    <div><label for="suggestion">Documentation suggestion</label></div>
-    <textarea id="suggestion" rows="8" cols="60" name="suggestion"></textarea>
-    <div data-sk-error="suggestion" class="form-error"></div>
-    <input id="_page" type="hidden" name="_page">
-    <input id="_subject" type="hidden" name="_subject" value="borgmatic documentation suggestion">
-    <br />
-    <label for="email">Email address</label>
-    <div><input id="email" type="email" name="email" placeholder="Only required if you want a response!"></div>
-    <div data-sk-error="email" class="form-error"></div>
-    <br />
-    <div><button type="submit">Send</button></div>
-    <br />
-</form>
-
-<script>
-    document.getElementById('_page').value = window.location.href;
-    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
-  
-    sk('form', 'init', {
-        id: '1d536680ab96',
-        element: '#suggestion-form'
-    });
-</script>
-
-<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/components/suggestion-link.html

@@ -0,0 +1,17 @@
+<h2>Improve this documentation</h2>
+
+<p>Have an idea on how to make this documentation even better? Use our <a
+href="https://projects.torsion.org/witten/borgmatic/issues">issue tracker</a> to send your
+feedback!</p>
+
+<script>
+    document.getElementById('_page').value = window.location.href;
+    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
+  
+    sk('form', 'init', {
+        id: '1d536680ab96',
+        element: '#suggestion-form'
+    });
+</script>
+
+<script defer src="https://js.statickit.com/statickit.js"></script>

docs/_includes/layouts/base.njk

@@ -11,7 +11,6 @@
 {% include 'components/minilink.css' %}
 {% include 'components/toc.css' %}
 {% include 'components/info-blocks.css' %}
-{% include 'components/suggestion-form.css' %}
 {% include 'prism-theme.css' %}
 {% include 'asciinema.css' %}
 {% endset %}

docs/_includes/layouts/main.njk

@@ -28,5 +28,5 @@ headerClass: elv-header-default
 
     {{ content | safe }}
 
-    {% include 'components/suggestion-form.html' %}
+    {% include 'components/suggestion-link.html' %}
 </main>

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -16,9 +16,14 @@ But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
 server is offline, then you'll get an annoying error message and the overall
 borgmatic run will fail (even if individual repositories still complete).
 
+Another variant is when the source machine is only sometimes available for
+backups, e.g. a laptop where you want to skip backups when the battery falls
+below a certain level.
+
 So what if you want borgmatic to swallow the error of a missing drive
-or an offline server, and continue trucking along? That's where the concept of
-"soft failure" come in.
+or an offline server or a low battery—and exit gracefully? That's where the
+concept of "soft failure" come in.
+
 
 ## Soft failure command hooks
 
@@ -78,6 +83,17 @@ hooks:
       - ping -q -c 1 buddys-server.org > /dev/null || exit 75
 ```
 
+Or to only run backups if the battery level is high enough:
+
+```yaml
+hooks:
+    before_backup:
+      - is_battery_percent_at_least.sh 25
+```
+
+(Writing the battery script is left as an exercise to the reader.)
+
+
 ## Caveats and details
 
 There are some caveats you should be aware of with this feature.

docs/how-to/develop-on-borgmatic.md

@@ -3,7 +3,7 @@ title: How to develop on borgmatic
 eleventyNavigation:
   key: Develop on borgmatic
   parent: How-to guides
-  order: 11
+  order: 12
 ---
 ## Source code
 

docs/how-to/run-arbitrary-borg-commands.md

@@ -0,0 +1,94 @@
+---
+title: How to run arbitrary Borg commands
+eleventyNavigation:
+  key: Run arbitrary Borg commands
+  parent: How-to guides
+  order: 10
+---
+## Running Borg with borgmatic
+
+Borg has several commands (and options) that borgmatic does not currently
+support. Sometimes though, as a borgmatic user, you may find yourself wanting
+to take advantage of these off-the-beaten-path Borg features. You could of
+course drop down to running Borg directly. But then you'd give up all the
+niceties of your borgmatic configuration. You could file a [borgmatic
+ticket](https://torsion.org/borgmatic/#issues) or even a [pull
+request](https://torsion.org/borgmatic/#contributing) to add the feature. But
+what if you need it *now*?
+
+That's where borgmatic's support for running "arbitrary" Borg commands comes
+in. Running Borg commands with borgmatic takes advantage of the following, all
+based on your borgmatic configuration files or command-line arguments:
+
+ * configured repositories (automatically runs your Borg command once for each
+   one)
+ * local and remote Borg binary paths
+ * SSH settings and Borg environment variables
+ * lock wait settings
+ * verbosity
+
+
+### borg action
+
+The way you run Borg with borgmatic is via the `borg` action. Here's a simple
+example:
+
+```bash
+borgmatic borg break-lock
+```
+
+(No `borg` action in borgmatic? Time to upgrade!)
+
+This runs Borg's `break-lock` command once on each configured borgmatic
+repository. Notice how the repository isn't present in the specified Borg
+options, as that part is provided by borgmatic.
+
+You can also specify Borg options for relevant commands:
+
+```bash
+borgmatic borg list --progress
+```
+
+This runs Borg's `list` command once on each configured borgmatic
+repository. However, the native `borgmatic list` action should be preferred
+for most use.
+
+What if you only want to run Borg on a single configured borgmatic repository
+when you've got several configured? Not a problem.
+
+```bash
+borgmatic borg --repository repo.borg break-lock
+```
+
+And what about a single archive?
+
+```bash
+borgmatic borg --archive your-archive-name list
+```
+
+### Limitations
+
+borgmatic's `borg` action is not without limitations:
+
+ * The Borg command you want to run (`create`, `list`, etc.) *must* come first
+   after the `borg` action. If you have any other Borg options to specify,
+   provide them after. For instance, `borgmatic borg list --progress` will work,
+   but `borgmatic borg --progress list` will not.
+ * borgmatic supplies the repository/archive name to Borg for you (based on
+   your borgmatic configuration or the `borgmatic borg --repository`/`--archive`
+   arguments), so do not specify the repository/archive otherwise.
+ * The `borg` action will not currently work for any Borg commands like `borg
+   serve` that do not accept a repository/archive name.
+ * Do not specify any global borgmatic arguments to the right of the `borg`
+   action. (They will be passed to Borg instead of borgmatic.) If you have
+   global borgmatic arguments, specify them *before* the `borg` action.
+ * Unlike other borgmatic actions, you cannot combine the `borg` action with
+   other borgmatic actions. This is to prevent ambiguity in commands like
+   `borgmatic borg list`, in which `list` is both a valid Borg command and a
+   borgmatic action. In this case, only the Borg command is run.
+ * Unlike normal borgmatic actions that support JSON, the `borg` action will
+   not disable certain borgmatic logs to avoid interfering with JSON output.
+
+In general, this `borgmatic borg` feature should be considered an escape
+valve—a feature of second resort. In the long run, it's preferable to wrap
+Borg commands with borgmatic actions that can support them fully.

docs/how-to/set-up-backups.md

@@ -68,7 +68,7 @@ sudo pip3 install --upgrade borgmatic
 
 The main downside of a global install is that borgmatic is less cleanly
 separated from the rest of your Python software, and there's the theoretical
-possibility of libary conflicts. But if you're okay with that, for instance
+possibility of library conflicts. But if you're okay with that, for instance
 on a relatively dedicated system, then a global install can work out fine.
 
 
@@ -87,16 +87,17 @@ installing borgmatic:
  * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
+ * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup)
  * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
  * [virtualenv](https://virtualenv.pypa.io/en/stable/)
 
 
 ## Hosting providers
 
-Need somewhere to store your encrypted offsite backups? The following hosting
-providers include specific support for Borg/borgmatic. Using these links and
-services helps support borgmatic development and hosting. (These are referral
-links, but without any tracking scripts or cookies.)
+Need somewhere to store your encrypted off-site backups? The following hosting
+providers include specific support for Borg/borgmatic—and fund borgmatic
+development and hosting when you use these links to sign up. (These are
+referral links, but without any tracking scripts or cookies.)
 
 <ul>
  <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
@@ -104,6 +105,10 @@ links, but without any tracking scripts or cookies.)
  <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
+Additionally, [Hetzner](https://www.hetzner.com/storage/storage-box) has a
+compatible storage offering, but does not currently fund borgmatic
+development or hosting.
+
 ## Configuration
 
 After you install borgmatic, generate a sample configuration file:
@@ -139,8 +144,8 @@ configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your
 
 ### Encryption
 
-If you encrypt your Borg repository with a passphrase instead of a key file,
-you'll either need to set the borgmatic `encryption_passphrase` configuration
+If you encrypt your Borg repository with a passphrase or a key file, you'll
+either need to set the borgmatic `encryption_passphrase` configuration
 variable or set the `BORG_PASSPHRASE` environment variable. See the
 [repository encryption
 section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)
@@ -257,11 +262,19 @@ You can modify the cron file if you'd like to run borgmatic more or less frequen
 
 ### systemd
 
-If you're using systemd instead of cron to run jobs, download the [sample
-systemd service
+If you're using systemd instead of cron to run jobs, you can still configure
+borgmatic to run automatically.
+
+(If you installed borgmatic from [Other ways to
+install](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install),
+you may already have borgmatic systemd service and timer files. If so, you may
+be able to skip some of the steps below.)
+
+First, download the [sample systemd service
 file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
 and the [sample systemd timer
 file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
+
 Then, from the directory where you downloaded them:
 
 ```bash

docs/how-to/upgrade.md

@@ -3,7 +3,7 @@ title: How to upgrade borgmatic
 eleventyNavigation:
   key: Upgrade borgmatic
   parent: How-to guides
-  order: 10
+  order: 11
 ---
 ## Upgrading
 

sample/cron/borgmatic

@@ -1,3 +1,3 @@
 # You can drop this file into /etc/cron.d/ to run borgmatic nightly.
 
-0 3 * * * root PATH=$PATH:/usr/bin:/usr/local/bin /root/.local/bin/borgmatic --syslog-verbosity 1
+0 3 * * * root PATH=$PATH:/usr/bin:/usr/local/bin /root/.local/bin/borgmatic --verbosity -1 --syslog-verbosity 1

sample/systemd/borgmatic.service

@@ -2,14 +2,16 @@
 Description=borgmatic backup
 Wants=network-online.target
 After=network-online.target
+# Prevent borgmatic from running unless the machine is plugged into power. Remove this line if you
+# want to allow borgmatic to run anytime.
 ConditionACPower=true
 
 [Service]
 Type=oneshot
 
-# Security settings for systemd running as root
-# For more details about this settings check the systemd manuals
-# https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+# Security settings for systemd running as root, optional but recommended to improve security. You
+# can disable individual settings if they cause problems for your use case. For more details, see
+# the systemd manual: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
 LockPersonality=true
 # Certain borgmatic features like Healthchecks integration need MemoryDenyWriteExecute to be off.
 # But you can try setting it to "yes" for improved security if you don't use those features.
@@ -29,6 +31,7 @@ RestrictRealtime=yes
 RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
 # Restrict write access
 # Change to 'ProtectSystem=strict' and uncomment 'ProtectHome' to make the whole file
 # system read-only be default and uncomment 'ReadWritePaths' for the required write access.
@@ -54,4 +57,4 @@ LogRateLimitIntervalSec=0
 # Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and
 # dbus-user-session to be installed.
 ExecStartPre=sleep 1m
-ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --syslog-verbosity 1
+ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --verbosity -1 --syslog-verbosity 1

scripts/release

@@ -15,6 +15,12 @@ if [[ ! -f NEWS ]]; then
 fi
 
 version=$(head --lines=1 NEWS)
+
+if [[ $version =~ .*dev* ]]; then
+    echo "Refusing to release a dev version: $version"
+    exit 1
+fi
+
 git tag $version
 git push origin $version
 git push github $version
@@ -32,7 +38,8 @@ twine upload -r pypi dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
 escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
 curl --silent --request POST \
-    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \
+    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases" \
+    --header "Authorization: token $projects_token" \
     --header "Accept: application/json" \
     --header "Content-Type: application/json" \
     --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}"

scripts/run-full-tests

@@ -10,9 +10,11 @@
 
 set -e
 
-python -m pip install --upgrade pip==20.0.2
-pip install tox==3.14.3
+apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client
+# If certain dependencies of black are available in this version of Alpine, install them.
+apk add --no-cache py3-typed-ast py3-regex || true
+python3 -m pip install --upgrade pip==20.2.4 setuptools==50.3.2
+pip3 install tox==3.20.1
 export COVERAGE_FILE=/tmp/.coverage
-tox --workdir /tmp/.tox
-apk add --no-cache borgbackup postgresql-client mariadb-client
-tox --workdir /tmp/.tox -e end-to-end
+tox --workdir /tmp/.tox --sitepackages
+tox --workdir /tmp/.tox --sitepackages -e end-to-end

setup.cfg

@@ -1,5 +1,5 @@
 [metadata]
-description-file=README.md
+description_file=README.md
 
 [tool:pytest]
 testpaths = tests

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.12'
+VERSION = '1.5.18'
 
 
 setup(
@@ -30,9 +30,9 @@ setup(
     },
     obsoletes=['atticmatic'],
     install_requires=(
-        'pykwalify>=1.6.0,<14.06',
+        'jsonschema',
         'requests',
-        'ruamel.yaml>0.15.0,<0.17.0',
+        'ruamel.yaml>0.15.0,<0.18.0',
         'setuptools',
         'colorama>=0.4.1,<0.5',
     ),

test_requirements.txt

@@ -1,25 +1,23 @@
-appdirs==1.4.3
-atomicwrites==1.3.0
-attrs==19.3.0
-black==19.3b0; python_version >= '3.6'
-click==7.0
-colorama==0.4.1
-coverage==4.5.4
-docopt==0.6.2
-flake8==3.7.9
+appdirs==1.4.4; python_version >= '3.8'
+attrs==20.3.0; python_version >= '3.8'
+black==19.10b0; python_version >= '3.8'
+click==7.1.2; python_version >= '3.8'
+colorama==0.4.4
+coverage==5.3
+flake8==3.8.4
 flexmock==0.10.4
-isort==4.3.21
+isort==5.9.1
 mccabe==0.6.1
-more-itertools==7.2.0
-pluggy==0.13.0
-py==1.8.0
-pycodestyle==2.5.0
-pyflakes==2.1.1
-pykwalify==1.7.0
-pytest==5.2.2
-pytest-cov==2.8.1
-python-dateutil==2.8.0
-PyYAML==5.1.2
-requests==2.22.0
-ruamel.yaml>0.15.0,<0.17.0
-toml==0.10.0
+pluggy==0.13.1
+pathspec==0.8.1; python_version >= '3.8'
+py==1.10.0
+pycodestyle==2.6.0
+pyflakes==2.2.0
+jsonschema==3.2.0
+pytest==6.1.2
+pytest-cov==2.10.1
+regex; python_version >= '3.8'
+requests==2.25.0
+ruamel.yaml>0.15.0,<0.18.0
+toml==0.10.2; python_version >= '3.8'
+typed-ast==1.4.2; python_version >= '3.8'

tests/end-to-end/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:12.2-alpine
+    image: postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
@@ -11,7 +11,7 @@ services:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   tests:
-    image: python:3.8-alpine3.11
+    image: alpine:3.13
     volumes:
       - "../..:/app:ro"
     tmpfs:

tests/end-to-end/test_borgmatic.py

@@ -20,6 +20,7 @@ def generate_configuration(config_path, repository_path):
         .read()
         .replace('user@backupserver:sourcehostname.borg', repository_path)
         .replace('- user@backupserver:{fqdn}', '')
+        .replace('- /home/user/path with spaces', '')
         .replace('- /home', '- {}'.format(config_path))
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')

tests/integration/commands/test_arguments.py

@@ -163,6 +163,24 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys):
     assert 'create arguments:' in captured.out
 
 
+def test_parse_arguments_with_action_before_global_options_parses_options():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('prune', '--verbosity', '2')
+
+    assert 'prune' in arguments
+    assert arguments['global'].verbosity == 2
+
+
+def test_parse_arguments_with_global_options_before_action_parses_options():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    arguments = module.parse_arguments('--verbosity', '2', 'prune')
+
+    assert 'prune' in arguments
+    assert arguments['global'].verbosity == 2
+
+
 def test_parse_arguments_with_prune_action_leaves_other_actions_disabled():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/config/test_generate.py

@@ -122,38 +122,44 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
 
 def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
     config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
-    schema = {'seq': [{'type': 'str'}]}
+    schema = {'type': 'array', 'items': {'type': 'string'}}
 
     module.add_comments_to_configuration_sequence(config, schema)
 
 
 def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
     config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
-    schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]}
+    schema = {
+        'type': 'array',
+        'items': {'type': 'object', 'properties': {'foo': {'description': 'yo'}}},
+    }
 
     module.add_comments_to_configuration_sequence(config, schema)
 
 
 def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
     config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
-    schema = {'seq': [{'map': {'foo': {}}}]}
+    schema = {'type': 'array', 'items': {'type': 'object', 'properties': {'foo': {}}}}
 
     module.add_comments_to_configuration_sequence(config, schema)
 
 
-def test_add_comments_to_configuration_map_does_not_raise():
+def test_add_comments_to_configuration_object_does_not_raise():
     # Ensure that it can deal with fields both in the schema and missing from the schema.
     config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
-    schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
+    schema = {
+        'type': 'object',
+        'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}},
+    }
 
-    module.add_comments_to_configuration_map(config, schema)
+    module.add_comments_to_configuration_object(config, schema)
 
 
-def test_add_comments_to_configuration_map_with_skip_first_does_not_raise():
+def test_add_comments_to_configuration_object_with_skip_first_does_not_raise():
     config = module.yaml.comments.CommentedMap([('foo', 33)])
-    schema = {'map': {'foo': {'desc': 'Foo'}}}
+    schema = {'type': 'object', 'properties': {'foo': {'description': 'Foo'}}}
 
-    module.add_comments_to_configuration_map(config, schema, skip_first=True)
+    module.add_comments_to_configuration_object(config, schema, skip_first=True)
 
 
 def test_remove_commented_out_sentinel_keeps_other_comments():

tests/integration/test_execute.py

@@ -1,5 +1,6 @@
 import logging
 import subprocess
+import sys
 
 import pytest
 from flexmock import flexmock
@@ -88,13 +89,21 @@ def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdou
 
 def test_log_outputs_kills_other_processes_when_one_errors():
     flexmock(module.logger).should_receive('log')
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
     flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, None, 'borg'
+    ).and_return(False)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, 2, 'borg'
+    ).and_return(True)
     other_process = subprocess.Popen(
-        ['watch', 'true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        ['sleep', '2'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
     )
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        other_process, None, 'borg'
+    ).and_return(False)
     flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
         process.stdout
     )
@@ -115,13 +124,87 @@ def test_log_outputs_kills_other_processes_when_one_errors():
     assert error.value.output
 
 
+def test_log_outputs_vents_other_processes_when_one_exits():
+    '''
+    Execute a command to generate a longish random string and pipe it into another command that
+    exits quickly. The test is basically to ensure we don't hang forever waiting for the exited
+    process to read the pipe, and that the string-generating process eventually gets vented and
+    exits.
+    '''
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(
+        [
+            sys.executable,
+            '-c',
+            "import random, string; print(''.join(random.choice(string.ascii_letters) for _ in range(40000)))",
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    other_process = subprocess.Popen(
+        ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        process, (process.stdout,)
+    ).and_return(process.stderr)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        other_process, (process.stdout,)
+    ).and_return(other_process.stdout)
+    flexmock(process.stdout).should_call('readline').at_least().once()
+
+    module.log_outputs(
+        (process, other_process),
+        exclude_stdouts=(process.stdout,),
+        output_log_level=logging.INFO,
+        borg_local_path='borg',
+    )
+
+
+def test_log_outputs_does_not_error_when_one_process_exits():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(
+        [
+            sys.executable,
+            '-c',
+            "import random, string; print(''.join(random.choice(string.ascii_letters) for _ in range(40000)))",
+        ],
+        stdout=None,  # Specifically test the case of a process without stdout captured.
+        stderr=None,
+    )
+    other_process = subprocess.Popen(
+        ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        process, (process.stdout,)
+    ).and_return(process.stderr)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        other_process, (process.stdout,)
+    ).and_return(other_process.stdout)
+
+    module.log_outputs(
+        (process, other_process),
+        exclude_stdouts=(process.stdout,),
+        output_log_level=logging.INFO,
+        borg_local_path='borg',
+    )
+
+
 def test_log_outputs_truncates_long_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
     flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, None, 'borg'
+    ).and_return(False)
+    flexmock(module).should_receive('exit_code_indicates_error').with_args(
+        process, 2, 'borg'
+    ).and_return(True)
     flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     with pytest.raises(subprocess.CalledProcessError) as error:

tests/unit/borg/test_borg.py

@@ -0,0 +1,123 @@
+import logging
+
+from flexmock import flexmock
+
+from borgmatic.borg import borg as module
+
+from ..test_verbosity import insert_logging_mock
+
+
+def test_run_arbitrary_borg_calls_borg_with_parameters():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--info'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+    insert_logging_mock(logging.INFO)
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--debug', '--show-rc'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+    insert_logging_mock(logging.DEBUG)
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--lock-wait', '5'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config=storage_config, options=['break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter():
+    storage_config = {}
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo::archive'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config=storage_config, options=['break-lock'], archive='archive',
+    )
+
+
+def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'], local_path='borg1',
+    )
+
+
+def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['break-lock'], remote_path='borg1',
+    )
+
+
+def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo', '--progress'),
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['list', '--progress'],
+    )
+
+
+def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'break-lock', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=['--', 'break-lock'],
+    )
+
+
+def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg',), output_log_level=logging.WARNING, borg_local_path='borg',
+    )
+
+    module.run_arbitrary_borg(
+        repository='repo', storage_config={}, options=[],
+    )

tests/unit/commands/test_arguments.py

@@ -5,146 +5,137 @@ from borgmatic.commands import arguments as module
 
 def test_parse_subparser_arguments_consumes_subparser_arguments_before_subparser_name():
     action_namespace = flexmock(foo=True)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('--foo', 'true', 'action'), subparsers
     )
 
-    arguments = module.parse_subparser_arguments(('--foo', 'true', 'action'), subparsers)
-
     assert arguments == {'action': action_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_consumes_subparser_arguments_after_subparser_name():
     action_namespace = flexmock(foo=True)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(parse_known_args=lambda arguments: (action_namespace, [])),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(parse_known_args=lambda arguments: (action_namespace, ['action'])),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('action', '--foo', 'true'), subparsers
     )
 
-    arguments = module.parse_subparser_arguments(('action', '--foo', 'true'), subparsers)
-
     assert arguments == {'action': action_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_consumes_subparser_arguments_with_alias():
     action_namespace = flexmock(foo=True)
-    action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, []))
-    subparsers = flexmock(
-        choices={
-            'action': action_subparser,
-            '-a': action_subparser,
-            'other': flexmock(),
-            '-o': flexmock(),
-        }
-    )
+    action_subparser = flexmock(parse_known_args=lambda arguments: (action_namespace, ['action']))
+    subparsers = {
+        'action': action_subparser,
+        '-a': action_subparser,
+        'other': flexmock(),
+        '-o': flexmock(),
+    }
     flexmock(module).SUBPARSER_ALIASES = {'action': ['-a'], 'other': ['-o']}
 
-    arguments = module.parse_subparser_arguments(('-a', '--foo', 'true'), subparsers)
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('-a', '--foo', 'true'), subparsers
+    )
 
     assert arguments == {'action': action_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_consumes_multiple_subparser_arguments():
     action_namespace = flexmock(foo=True)
     other_namespace = flexmock(bar=3)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--bar', '3'])
-            ),
-            'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])),
-        }
-    )
+    subparsers = {
+        'action': flexmock(
+            parse_known_args=lambda arguments: (action_namespace, ['action', '--bar', '3'])
+        ),
+        'other': flexmock(parse_known_args=lambda arguments: (other_namespace, [])),
+    }
 
-    arguments = module.parse_subparser_arguments(
+    arguments, remaining_arguments = module.parse_subparser_arguments(
         ('action', '--foo', 'true', 'other', '--bar', '3'), subparsers
     )
 
     assert arguments == {'action': action_namespace, 'other': other_namespace}
+    assert remaining_arguments == []
 
 
 def test_parse_subparser_arguments_applies_default_subparsers():
     prune_namespace = flexmock()
     create_namespace = flexmock(progress=True)
     check_namespace = flexmock()
-    subparsers = flexmock(
-        choices={
-            'prune': flexmock(parse_known_args=lambda arguments: (prune_namespace, ['--progress'])),
-            'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])),
-            'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])),
-            'other': flexmock(),
-        }
-    )
+    subparsers = {
+        'prune': flexmock(
+            parse_known_args=lambda arguments: (prune_namespace, ['prune', '--progress'])
+        ),
+        'create': flexmock(parse_known_args=lambda arguments: (create_namespace, [])),
+        'check': flexmock(parse_known_args=lambda arguments: (check_namespace, [])),
+        'other': flexmock(),
+    }
 
-    arguments = module.parse_subparser_arguments(('--progress'), subparsers)
+    arguments, remaining_arguments = module.parse_subparser_arguments(('--progress'), subparsers)
 
     assert arguments == {
         'prune': prune_namespace,
         'create': create_namespace,
         'check': check_namespace,
     }
+    assert remaining_arguments == []
 
 
-def test_parse_global_arguments_with_help_does_not_apply_default_subparsers():
-    global_namespace = flexmock(verbosity='lots')
+def test_parse_subparser_arguments_passes_through_unknown_arguments_before_subparser_name():
     action_namespace = flexmock()
-    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
-            ),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(
+            parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots'])
+        ),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('--verbosity', 'lots', 'action'), subparsers
     )
 
-    arguments = module.parse_global_arguments(
-        ('--verbosity', 'lots', '--help'), top_level_parser, subparsers
-    )
-
-    assert arguments == global_namespace
+    assert arguments == {'action': action_namespace}
+    assert remaining_arguments == ['--verbosity', 'lots']
 
 
-def test_parse_global_arguments_consumes_global_arguments_before_subparser_name():
-    global_namespace = flexmock(verbosity='lots')
+def test_parse_subparser_arguments_passes_through_unknown_arguments_after_subparser_name():
     action_namespace = flexmock()
-    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
-            ),
-            'other': flexmock(),
-        }
+    subparsers = {
+        'action': flexmock(
+            parse_known_args=lambda arguments: (action_namespace, ['action', '--verbosity', 'lots'])
+        ),
+        'other': flexmock(),
+    }
+
+    arguments, remaining_arguments = module.parse_subparser_arguments(
+        ('action', '--verbosity', 'lots'), subparsers
     )
 
-    arguments = module.parse_global_arguments(
-        ('--verbosity', 'lots', 'action'), top_level_parser, subparsers
-    )
-
-    assert arguments == global_namespace
+    assert arguments == {'action': action_namespace}
+    assert remaining_arguments == ['--verbosity', 'lots']
 
 
-def test_parse_global_arguments_consumes_global_arguments_after_subparser_name():
-    global_namespace = flexmock(verbosity='lots')
-    action_namespace = flexmock()
-    top_level_parser = flexmock(parse_args=lambda arguments: global_namespace)
-    subparsers = flexmock(
-        choices={
-            'action': flexmock(
-                parse_known_args=lambda arguments: (action_namespace, ['--verbosity', 'lots'])
-            ),
-            'other': flexmock(),
-        }
-    )
+def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparsers():
+    action_namespace = flexmock(options=[])
+    subparsers = {
+        'borg': flexmock(parse_known_args=lambda arguments: (action_namespace, ['borg', 'list'])),
+        'list': flexmock(),
+    }
 
-    arguments = module.parse_global_arguments(
-        ('action', '--verbosity', 'lots'), top_level_parser, subparsers
-    )
+    arguments, remaining_arguments = module.parse_subparser_arguments(('borg', 'list'), subparsers)
 
-    assert arguments == global_namespace
+    assert arguments == {'borg': action_namespace}
+    assert arguments['borg'].options == ['list']
+    assert remaining_arguments == []

tests/unit/commands/test_borgmatic.py

@@ -447,7 +447,9 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
         ['baz']
     )
-    flexmock(module.sys.stdout).should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    stdout = flexmock()
+    stdout.should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    flexmock(module.sys).stdout = stdout
     arguments = {}
 
     tuple(

tests/unit/config/test_convert.py

@@ -12,7 +12,7 @@ Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention',
 def test_convert_section_generates_integer_value_for_integer_type_in_schema():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
     source_section_config = OrderedDict([('check_last', '3')])
-    section_schema = {'map': {'check_last': {'type': 'int'}}}
+    section_schema = {'type': 'object', 'properties': {'check_last': {'type': 'integer'}}}
 
     destination_config = module._convert_section(source_section_config, section_schema)
 
@@ -21,7 +21,7 @@ def test_convert_section_generates_integer_value_for_integer_type_in_schema():
 
 def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module.generate).should_receive('add_comments_to_configuration_map')
+    flexmock(module.generate).should_receive('add_comments_to_configuration_object')
     source_config = Parsed_config(
         location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
         storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
@@ -29,7 +29,10 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
         consistency=OrderedDict([('checks', 'repository')]),
     )
     source_excludes = ['/var']
-    schema = {'map': defaultdict(lambda: {'map': {}})}
+    schema = {
+        'type': 'object',
+        'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}),
+    }
 
     destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)
 
@@ -54,7 +57,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
 
 def test_convert_legacy_parsed_config_splits_space_separated_values():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module.generate).should_receive('add_comments_to_configuration_map')
+    flexmock(module.generate).should_receive('add_comments_to_configuration_object')
     source_config = Parsed_config(
         location=OrderedDict(
             [('source_directories', '/home /etc'), ('repository', 'hostname.borg')]
@@ -64,7 +67,10 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
         consistency=OrderedDict([('checks', 'repository archives')]),
     )
     source_excludes = ['/var']
-    schema = {'map': defaultdict(lambda: {'map': {}})}
+    schema = {
+        'type': 'object',
+        'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}),
+    }
 
     destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)
 

tests/unit/config/test_generate.py

@@ -8,24 +8,32 @@ from borgmatic.config import generate as module
 
 def test_schema_to_sample_configuration_generates_config_map_with_examples():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module).should_receive('add_comments_to_configuration_map')
+    flexmock(module).should_receive('add_comments_to_configuration_object')
     schema = {
-        'map': OrderedDict(
+        'type': 'object',
+        'properties': OrderedDict(
             [
-                ('section1', {'map': {'field1': OrderedDict([('example', 'Example 1')])}}),
+                (
+                    'section1',
+                    {
+                        'type': 'object',
+                        'properties': {'field1': OrderedDict([('example', 'Example 1')])},
+                    },
+                ),
                 (
                     'section2',
                     {
-                        'map': OrderedDict(
+                        'type': 'object',
+                        'properties': OrderedDict(
                             [
                                 ('field2', {'example': 'Example 2'}),
                                 ('field3', {'example': 'Example 3'}),
                             ]
-                        )
+                        ),
                     },
                 ),
             ]
-        )
+        ),
     }
 
     config = module._schema_to_sample_configuration(schema)
@@ -41,7 +49,7 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples():
 def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example():
     flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
     flexmock(module).should_receive('add_comments_to_configuration_sequence')
-    schema = {'seq': [{'type': 'str'}], 'example': ['hi']}
+    schema = {'type': 'array', 'items': {'type': 'string'}, 'example': ['hi']}
 
     config = module._schema_to_sample_configuration(schema)
 
@@ -51,15 +59,15 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_strings_wit
 def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples():
     flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
     flexmock(module).should_receive('add_comments_to_configuration_sequence')
-    flexmock(module).should_receive('add_comments_to_configuration_map')
+    flexmock(module).should_receive('add_comments_to_configuration_object')
     schema = {
-        'seq': [
-            {
-                'map': OrderedDict(
-                    [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
-                )
-            }
-        ]
+        'type': 'array',
+        'items': {
+            'type': 'object',
+            'properties': OrderedDict(
+                [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
+            ),
+        },
     }
 
     config = module._schema_to_sample_configuration(schema)

tests/unit/config/test_validate.py

@@ -4,8 +4,33 @@ from flexmock import flexmock
 from borgmatic.config import validate as module
 
 
-def test_validation_error_str_contains_error_messages_and_config_filename():
-    error = module.Validation_error('config.yaml', ('oops', 'uh oh'))
+def test_format_error_path_element_formats_array_index():
+    module.format_error_path_element(3) == '[3]'
+
+
+def test_format_error_path_element_formats_property():
+    module.format_error_path_element('foo') == '.foo'
+
+
+def test_format_error_formats_error_including_path():
+    flexmock(module).format_error_path_element = lambda element: '.{}'.format(element)
+    error = flexmock(message='oops', path=['foo', 'bar'])
+
+    assert module.format_error(error) == "At 'foo.bar': oops"
+
+
+def test_format_error_formats_error_without_path():
+    flexmock(module).should_receive('format_error_path_element').never()
+    error = flexmock(message='oops', path=[])
+
+    assert module.format_error(error) == 'At the top level: oops'
+
+
+def test_validation_error_string_contains_error_messages_and_config_filename():
+    flexmock(module).format_error = lambda error: error.message
+    error = module.Validation_error(
+        'config.yaml', (flexmock(message='oops', path=None), flexmock(message='uh oh'))
+    )
 
     result = str(error)
 
@@ -15,6 +40,8 @@ def test_validation_error_str_contains_error_messages_and_config_filename():
 
 
 def test_apply_logical_validation_raises_if_archive_name_format_present_without_prefix():
+    flexmock(module).format_error = lambda error: error.message
+
     with pytest.raises(module.Validation_error):
         module.apply_logical_validation(
             'config.yaml',
@@ -26,6 +53,8 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
 
 
 def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix():
+    flexmock(module).format_error = lambda error: error.message
+
     with pytest.raises(module.Validation_error):
         module.apply_logical_validation(
             'config.yaml',
@@ -38,6 +67,8 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
 
 
 def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories():
+    flexmock(module).format_error = lambda error: error.message
+
     with pytest.raises(module.Validation_error):
         module.apply_logical_validation(
             'config.yaml',
@@ -75,27 +106,6 @@ def test_apply_logical_validation_does_not_raise_otherwise():
     module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}})
 
 
-def test_remove_examples_strips_examples_from_map():
-    schema = {
-        'map': {
-            'foo': {'desc': 'thing1', 'example': 'bar'},
-            'baz': {'desc': 'thing2', 'example': 'quux'},
-        }
-    }
-
-    module.remove_examples(schema)
-
-    assert schema == {'map': {'foo': {'desc': 'thing1'}, 'baz': {'desc': 'thing2'}}}
-
-
-def test_remove_examples_strips_examples_from_sequence_of_maps():
-    schema = {'seq': [{'map': {'foo': {'desc': 'thing', 'example': 'bar'}}, 'example': 'stuff'}]}
-
-    module.remove_examples(schema)
-
-    assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]}
-
-
 def test_normalize_repository_path_passes_through_remote_repository():
     repository = 'example.org:test.borg'
 

tests/unit/hooks/test_healthchecks.py

@@ -28,7 +28,9 @@ def test_forgetful_buffering_handler_emit_forgets_log_records_when_capacity_reac
 def test_format_buffered_logs_for_payload_flattens_log_buffer():
     handler = module.Forgetful_buffering_handler(byte_capacity=100, log_level=1)
     handler.buffer = ['foo\n', 'bar\n']
-    flexmock(module.logging).should_receive('getLogger').and_return(flexmock(handlers=[handler]))
+    logger = flexmock(handlers=[handler])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 
@@ -39,7 +41,9 @@ def test_format_buffered_logs_for_payload_inserts_truncation_indicator_when_logs
     handler = module.Forgetful_buffering_handler(byte_capacity=100, log_level=1)
     handler.buffer = ['foo\n', 'bar\n']
     handler.forgot = True
-    flexmock(module.logging).should_receive('getLogger').and_return(flexmock(handlers=[handler]))
+    logger = flexmock(handlers=[handler])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 
@@ -47,9 +51,9 @@ def test_format_buffered_logs_for_payload_inserts_truncation_indicator_when_logs
 
 
 def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload():
-    flexmock(module.logging).should_receive('getLogger').and_return(
-        flexmock(handlers=[module.logging.Handler()])
-    )
+    logger = flexmock(handlers=[module.logging.Handler()])
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
 
     payload = module.format_buffered_logs_for_payload()
 

tests/unit/test_execute.py

@@ -21,6 +21,11 @@ from borgmatic import execute as module
         (flexmock(args=['grep']), 0, 'borg', False),
         (flexmock(args=['borg']), 0, 'borg', False),
         (flexmock(args=['borg1']), 0, 'borg1', False),
+        # -9 exit code occurs when child process get SIGKILLed.
+        (flexmock(args=['grep']), -9, None, True),
+        (flexmock(args=['grep']), -9, 'borg', True),
+        (flexmock(args=['borg']), -9, 'borg', True),
+        (flexmock(args=['borg1']), -9, 'borg1', True),
         (flexmock(args=['borg']), None, None, False),
     ),
 )

tests/unit/test_logger.py

@@ -179,6 +179,26 @@ def test_configure_logging_probes_for_log_socket_on_macos():
     module.configure_logging(logging.INFO)
 
 
+def test_configure_logging_probes_for_log_socket_on_freebsd():
+    flexmock(module).should_receive('Multi_stream_handler').and_return(
+        flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None)
+    )
+    flexmock(module).should_receive('Console_color_formatter')
+    flexmock(module).should_receive('interactive_console').and_return(False)
+    flexmock(module.logging).should_receive('basicConfig').with_args(
+        level=logging.INFO, handlers=tuple
+    )
+    flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(False)
+    flexmock(module.os.path).should_receive('exists').with_args('/var/run/syslog').and_return(False)
+    flexmock(module.os.path).should_receive('exists').with_args('/var/run/log').and_return(True)
+    syslog_handler = logging.handlers.SysLogHandler()
+    flexmock(module.logging.handlers).should_receive('SysLogHandler').with_args(
+        address='/var/run/log'
+    ).and_return(syslog_handler).once()
+
+    module.configure_logging(logging.INFO)
+
+
 def test_configure_logging_sets_global_logger_to_most_verbose_log_level():
     flexmock(module).should_receive('Multi_stream_handler').and_return(
         flexmock(setFormatter=lambda formatter: None, setLevel=lambda level: None)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35,py36,py37,py38
+envlist = py36,py37,py38,py39
 skip_missing_interpreters = True
 skipsdist = True
 minversion = 3.14.1
@@ -13,8 +13,8 @@ whitelist_externals =
 passenv = COVERAGE_FILE
 commands =
     pytest {posargs}
-    py36,py37,py38: black --check .
-    isort --recursive --check-only --settings-path setup.cfg .
+    py38,py39: black --check .
+    isort --check-only --settings-path setup.cfg .
     flake8 borgmatic tests
 
 [testenv:black]
@@ -34,4 +34,4 @@ commands =
 [testenv:isort]
 deps = {[testenv]deps}
 commands =
-    isort {posargs:--recursive} --settings-path setup.cfg .
+    isort --settings-path setup.cfg .