Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1096

NEWS

@@ -1,3 +1,18 @@
+2.0.6
+ * #1068: Fix a warning from LVM about leaked file descriptors.
+ * #1086: Fix for the "spot" check breaking when the "--progress" flag is used.
+ * #1089: Fix for the "spot" check erroring when a checksum command errors.
+ * #1091: Fix for the "config generate" action generating invalid configuration when upgrading
+   deprecated command hooks.
+ * #1093: Fix for the LVM hook erroring when the "--dry-run" flag is used.
+ * #1094: Fix incorrect documentation about customizing Borg exit codes:
+   https://torsion.org/borgmatic/docs/how-to/customize-warnings-and-errors/
+ * #1095: Fix for the "spot" check's "xxh64sum_command" option erroring on commands containing
+   spaces.
+ * Add support for Borg 2's "s3:" and "b2:" repository URLs, so you can backup to S3 or B2 cloud
+   storage services even without using Rclone.
+ * During the "spot" check, truncate log messages containing many file paths.
+
 2.0.5
  * #1033: Add a "password_transport" option to the MariaDB and MySQL database hooks for customizing
    how borgmatic transmits passwords to the database client.

borgmatic/actions/check.py

@@ -6,7 +6,9 @@ import logging
 import os
 import pathlib
 import random
+import shlex
 import shutil
+import textwrap
 
 import borgmatic.actions.pattern
 import borgmatic.borg.check
@@ -372,7 +374,11 @@ def collect_spot_check_source_paths(
         borgmatic.borg.create.make_base_create_command(
             dry_run=True,
             repository_path=repository['path'],
-            config=dict(config, list_details=True),
+            # Omit "progress" because it interferes with "list_details".
+            config=dict(
+                {option: value for option, value in config.items() if option != 'progress'},
+                list_details=True,
+            ),
             patterns=borgmatic.actions.pattern.process_patterns(
                 borgmatic.actions.pattern.collect_patterns(config),
                 working_directory,
@@ -454,7 +460,7 @@ def collect_spot_check_archive_paths(
     )
 
 
-SAMPLE_PATHS_SUBSET_COUNT = 10000
+SAMPLE_PATHS_SUBSET_COUNT = 5000
 
 
 def compare_spot_check_hashes(
@@ -508,7 +514,10 @@ def compare_spot_check_hashes(
             break
 
         hash_output = borgmatic.execute.execute_command_and_capture_output(
-            (spot_check_config.get('xxh64sum_command', 'xxh64sum'),)
+            tuple(
+                shlex.quote(part)
+                for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
+            )
             + tuple(
                 path for path in source_sample_paths_subset if path in hashable_source_sample_path
             ),
@@ -562,6 +571,9 @@ def compare_spot_check_hashes(
     return tuple(failing_paths)
 
 
+MAX_SPOT_CHECK_PATHS_LENGTH = 1000
+
+
 def spot_check(
     repository,
     config,
@@ -629,9 +641,12 @@ def spot_check(
     logger.debug(f'{len(archive_paths)} total archive paths for spot check')
 
     if len(source_paths) == 0:
-        logger.debug(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}'
+        truncated_archive_paths = textwrap.shorten(
+            ', '.join(set(archive_paths)) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
+        logger.debug(f'Paths in latest archive but not source paths: {truncated_archive_paths}')
         raise ValueError(
             'Spot check failed: There are no source paths to compare against the archive'
         )
@@ -642,11 +657,21 @@ def spot_check(
 
     if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
         rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
-        logger.debug(
-            f'Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}'
+        truncated_exclusive_source_paths = textwrap.shorten(
+            ', '.join(rootless_source_paths - set(archive_paths)) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
         )
         logger.debug(
-            f'Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}'
+            f'Paths in source paths but not latest archive: {truncated_exclusive_source_paths}'
+        )
+        truncated_exclusive_archive_paths = textwrap.shorten(
+            ', '.join(set(archive_paths) - rootless_source_paths) or 'none',
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
+        )
+        logger.debug(
+            f'Paths in latest archive but not source paths: {truncated_exclusive_archive_paths}'
         )
         raise ValueError(
             f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
@@ -669,8 +694,13 @@ def spot_check(
     failing_percentage = (len(failing_paths) / len(source_paths)) * 100
 
     if failing_percentage > data_tolerance_percentage:
+        truncated_failing_paths = textwrap.shorten(
+            ', '.join(failing_paths),
+            width=MAX_SPOT_CHECK_PATHS_LENGTH,
+            placeholder=' ...',
+        )
         logger.debug(
-            f'Source paths with data not matching the latest archive: {", ".join(failing_paths)}'
+            f'Source paths with data not matching the latest archive: {truncated_failing_paths}'
         )
         raise ValueError(
             f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'

borgmatic/config/generate.py

@@ -51,7 +51,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
         add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
     elif borgmatic.config.schema.compare_types(schema_type, {'object'}):
         if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict):
-            source_config = dict(collections.ChainMap(*source_config))
+            source_config = source_config[0]
 
         config = (
             ruamel.yaml.comments.CommentedMap(
@@ -71,7 +71,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
         add_comments_to_configuration_object(
-            config, schema, source_config, indent=indent, skip_first=parent_is_sequence
+            config, schema, source_config, indent=indent, skip_first_field=parent_is_sequence
         )
     elif borgmatic.config.schema.compare_types(schema_type, SCALAR_SCHEMA_TYPES, match=all):
         return example
@@ -108,17 +108,30 @@ def comment_out_optional_configuration(rendered_config):
     '''
     lines = []
     optional = False
+    indent_characters = None
+    indent_characters_at_sentinel = None
 
     for line in rendered_config.split('\n'):
+        indent_characters = len(line) - len(line.lstrip())
+
         # Upon encountering an optional configuration option, comment out lines until the next blank
         # line.
         if line.strip().startswith(f'# {COMMENTED_OUT_SENTINEL}'):
             optional = True
+            indent_characters_at_sentinel = indent_characters
             continue
 
         # Hit a blank line, so reset commenting.
         if not line.strip():
             optional = False
+            indent_characters_at_sentinel = None
+        # Dedented, so reset commenting.
+        elif (
+            indent_characters_at_sentinel is not None
+            and indent_characters < indent_characters_at_sentinel
+        ):
+            optional = False
+            indent_characters_at_sentinel = None
 
         lines.append(comment_out_line(line) if optional else line)
 
@@ -198,25 +211,28 @@ COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
 
 
 def add_comments_to_configuration_object(
-    config, schema, source_config=None, indent=0, skip_first=False
+    config, schema, source_config=None, indent=0, skip_first_field=False
 ):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
     configuration dict, putting them before each field. Indent the comment the given number of
     characters.
 
+    If skip_first_field is True, omit the comment for the initial field. This is useful for
+    sequences, where the comment for the first field goes before the sequence itself.
+
     And a sentinel for commenting out options that are neither in DEFAULT_KEYS nor the the given
     source configuration dict. The idea is that any options used in the source configuration should
     stay active in the generated configuration.
     '''
     for index, field_name in enumerate(config.keys()):
-        if skip_first and index == 0:
+        if skip_first_field and index == 0:
             continue
 
         field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {})
         description = field_schema.get('description', '').strip()
 
-        # If this isn't a default key, add an indicator to the comment flagging it to be commented
+        # If this isn't a default key, add an indicator to the comment, flagging it to be commented
         # out from the sample configuration. This sentinel is consumed by downstream processing that
         # does the actual commenting out.
         if field_name not in DEFAULT_KEYS and (
@@ -299,6 +315,12 @@ def generate_sample_configuration(
         source_config = load.load_configuration(source_filename)
         normalize.normalize(source_filename, source_config)
 
+        # The borgmatic.config.normalize.normalize() function tacks on an empty "bootstrap" if
+        # needed, so the hook gets used by default. But we don't want it to end up in the generated
+        # config unless the user has set it explicitly, as an empty "bootstrap:" won't validate.
+        if source_config and source_config.get('bootstrap') == {}:
+            del source_config['bootstrap']
+
     destination_config = merge_source_configuration_into_destination(
         schema_to_sample_configuration(schema, source_config), source_config
     )

borgmatic/config/normalize.py

@@ -159,7 +159,7 @@ def normalize(config_filename, config):
                 dict(
                     levelno=logging.WARNING,
                     levelname='WARNING',
-                    msg=f'{config_filename}: The borgmatic_source_directory option is deprecated and will be removed from a future release. Use borgmatic_runtime_directory and borgmatic_state_directory instead.',
+                    msg=f'{config_filename}: The borgmatic_source_directory option is deprecated and will be removed from a future release. Use user_runtime_directory and user_state_directory instead.',
                 )
             )
         )
@@ -356,6 +356,8 @@ def normalize(config_filename, config):
                     repository_path.startswith('ssh://')
                     or repository_path.startswith('sftp://')
                     or repository_path.startswith('rclone:')
+                    or repository_path.startswith('s3:')
+                    or repository_path.startswith('b2:')
                 ):
                     config['repositories'].append(repository_dict)
                 else:
@@ -365,7 +367,7 @@ def normalize(config_filename, config):
                             dict(
                                 levelno=logging.WARNING,
                                 levelname='WARNING',
-                                msg=f'{config_filename}: Remote repository paths without ssh:// or rclone: syntax are deprecated and support will be removed from a future release. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
+                                msg=f'{config_filename}: Remote repository paths without ssh://, sftp://, rclone:, s3:, or b2:, syntax are deprecated and support will be removed from a future release. Interpreting "{repository_path}" as "{rewritten_repository_path}"',
                             )
                         )
                     )

borgmatic/execute.py

@@ -288,6 +288,7 @@ def execute_command(
     borg_local_path=None,
     borg_exit_codes=None,
     run_to_completion=True,
+    close_fds=False,  # Necessary for passing credentials via anonymous pipe.
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
@@ -315,8 +316,7 @@ def execute_command(
         shell=shell,
         env=environment,
         cwd=working_directory,
-        # Necessary for passing credentials via anonymous pipe.
-        close_fds=False,
+        close_fds=close_fds,
     )
     if not run_to_completion:
         return process
@@ -340,6 +340,7 @@ def execute_command_and_capture_output(
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
+    close_fds=False,  # Necessary for passing credentials via anonymous pipe.
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its
@@ -365,8 +366,7 @@ def execute_command_and_capture_output(
             shell=shell,
             env=environment,
             cwd=working_directory,
-            # Necessary for passing credentials via anonymous pipe.
-            close_fds=False,
+            close_fds=close_fds,
         )
     except subprocess.CalledProcessError as error:
         if (
@@ -390,6 +390,7 @@ def execute_command_with_processes(
     working_directory=None,
     borg_local_path=None,
     borg_exit_codes=None,
+    close_fds=False,  # Necessary for passing credentials via anonymous pipe.
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
@@ -425,8 +426,7 @@ def execute_command_with_processes(
             shell=shell,
             env=environment,
             cwd=working_directory,
-            # Necessary for passing credentials via anonymous pipe.
-            close_fds=False,
+            close_fds=close_fds,
         )
     except (subprocess.CalledProcessError, OSError):
         # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.

borgmatic/hooks/data_source/btrfs.py

@@ -32,7 +32,8 @@ def get_subvolume_mount_points(findmnt_command):
             'btrfs',
             '--json',
             '--list',  # Request a flat list instead of a nested subvolume hierarchy.
-        )
+        ),
+        close_fds=True,
     )
 
     try:
@@ -59,6 +60,7 @@ def get_subvolume_property(btrfs_command, subvolume_path, property_name):
             subvolume_path,
             property_name,
         ),
+        close_fds=True,
     )
 
     try:
@@ -227,6 +229,7 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma:
             snapshot_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -298,6 +301,7 @@ def delete_snapshot(btrfs_command, snapshot_path):  # pragma: no cover
             snapshot_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 

borgmatic/hooks/data_source/lvm.py

@@ -50,7 +50,8 @@ def get_logical_volumes(lsblk_command, patterns=None):
                     'name,path,mountpoint,type',
                     '--json',
                     '--list',
-                )
+                ),
+                close_fds=True,
             )
         )
     except json.JSONDecodeError as error:
@@ -109,6 +110,7 @@ def snapshot_logical_volume(
             logical_volume_device,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -129,6 +131,7 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # prag
             snapshot_mount_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -227,12 +230,13 @@ def dump_data_sources(
             )
 
         # Get the device path for the snapshot we just created.
-        try:
-            snapshot = get_snapshots(
-                hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name
-            )[0]
-        except IndexError:
-            raise ValueError(f'Cannot find LVM snapshot {snapshot_name}')
+        if not dry_run:
+            try:
+                snapshot = get_snapshots(
+                    hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name
+                )[0]
+            except IndexError:
+                raise ValueError(f'Cannot find LVM snapshot {snapshot_name}')
 
         # Mount the snapshot into a particular named temporary directory so that the snapshot ends
         # up in the Borg archive at the "original" logical volume mount point path.
@@ -277,6 +281,7 @@ def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
     borgmatic.execute.execute_command(
         tuple(umount_command.split(' ')) + (snapshot_mount_path,),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -291,6 +296,7 @@ def remove_snapshot(lvremove_command, snapshot_device_path):  # pragma: no cover
             snapshot_device_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -318,7 +324,8 @@ def get_snapshots(lvs_command, snapshot_name=None):
                     'lv_name,lv_path',
                     '--select',
                     'lv_attr =~ ^s',  # Filter to just snapshots.
-                )
+                ),
+                close_fds=True,
             )
         )
     except json.JSONDecodeError as error:

borgmatic/hooks/data_source/zfs.py

@@ -53,7 +53,8 @@ def get_datasets_to_backup(zfs_command, patterns):
             'filesystem',
             '-o',
             f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}',
-        )
+        ),
+        close_fds=True,
     )
 
     try:
@@ -131,7 +132,8 @@ def get_all_dataset_mount_points(zfs_command):
             'filesystem',
             '-o',
             'mountpoint',
-        )
+        ),
+        close_fds=True,
     )
 
     return tuple(
@@ -158,6 +160,7 @@ def snapshot_dataset(zfs_command, full_snapshot_name):  # pragma: no cover
             full_snapshot_name,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -180,6 +183,7 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
             snapshot_mount_path,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -310,6 +314,7 @@ def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
     borgmatic.execute.execute_command(
         tuple(umount_command.split(' ')) + (snapshot_mount_path,),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -325,6 +330,7 @@ def destroy_snapshot(zfs_command, full_snapshot_name):  # pragma: no cover
             full_snapshot_name,
         ),
         output_log_level=logging.DEBUG,
+        close_fds=True,
     )
 
 
@@ -342,7 +348,8 @@ def get_all_snapshots(zfs_command):
             'snapshot',
             '-o',
             'name',
-        )
+        ),
+        close_fds=True,
     )
 
     return tuple(line.rstrip() for line in list_output.splitlines())

docs/how-to/customize-warnings-and-errors.md

@@ -12,52 +12,18 @@ numeric ID indicating success, warning, or error. borgmatic consumes this exit
 code to decide how to respond. Normally, a Borg error results in a borgmatic
 error, while a Borg warning or success doesn't.
 
-But if that default behavior isn't sufficient for your needs, you can
-customize how borgmatic interprets [Borg's exit
+<span class="minilink minilink-addedin">With Borg version 1.4+</span> If the
+default behavior isn't sufficient for your needs, you can customize how
+borgmatic interprets [Borg's exit
 codes](https://borgbackup.readthedocs.io/en/stable/usage/general.html#return-codes).
-For instance, to elevate Borg warnings to errors, thereby causing borgmatic to
-error on them, use the following borgmatic configuration:
-
-```yaml
-borg_exit_codes:
-   - exit_code: 1
-     treat_as: error
-```
-
-Be aware though that Borg exits with a warning code for a variety of benign
-situations such as files changing while they're being read, so this example
-may not meet your needs. Keep reading though for more granular exit code
-configuration.
-
-Here's an example that squashes Borg errors to warnings:
-
-```yaml
-borg_exit_codes:
-   - exit_code: 2
-     treat_as: warning
-```
-
-Be careful with this example though, because it prevents borgmatic from
-erroring when Borg errors, which may not be desirable.
-
-
-### More granular configuration
-
-<span class="minilink minilink-addedin">New in Borg version 1.4</span> Borg
-support for [more granular exit
-codes](https://borgbackup.readthedocs.io/en/1.4-maint/usage/general.html#return-codes)
-means that you can configure borgmatic to respond to specific Borg conditions.
-See the full list of [Borg 1.4 error and warning exit
-codes](https://borgbackup.readthedocs.io/en/stable/internals/frontends.html#message-ids).
-The `rc:` numeric value there tells you the exit code for each.
 
 For instance, this borgmatic configuration elevates all Borg backup file
 permission warnings (exit code `105`)—and only those warnings—to errors:
 
 ```yaml
 borg_exit_codes:
-   - exit_code: 105
-     treat_as: error
+    - code: 105
+      treat_as: error
 ```
 
 The following configuration does that *and* elevates backup file not found
@@ -65,16 +31,20 @@ warnings (exit code `107`) to errors as well:
 
 ```yaml
 borg_exit_codes:
-   - exit_code: 105
-     treat_as: error
-   - exit_code: 107
-     treat_as: error
+    - code: 105
+      treat_as: error
+    - code: 107
+      treat_as: error
 ```
 
+See the full list of [Borg 1.4 error and warning exit
+codes](https://borgbackup.readthedocs.io/en/stable/internals/frontends.html#message-ids).
+The `rc:` numeric value there tells you the exit code for each.
+
 If you don't know the exit code for a particular Borg error or warning you're
-experiencing, you can usually find it in your borgmatic output when
-`--verbosity 2` is enabled. For instance, here's a snippet of that output when
-a backup file is not found:
+experiencing, you can usually find it in your borgmatic output when `--verbosity
+2` is enabled. For instance, here's a snippet of that output when a backup file
+is not found:
 
 ```
 /noexist: stat: [Errno 2] No such file or directory: '/noexist'
@@ -84,3 +54,30 @@ terminating with warning status, rc 107
 
 So if you want to configure borgmatic to treat this as an error instead of a
 warning, the exit status to use is `107`.
+
+<span class="minilink minilink-addedin">With Borg version 1.2 and earlier</span>
+Older versions of Borg didn't support granular exit codes, but still
+distinguished between Borg errors and warnings. For instance, to elevate Borg
+warnings to errors, thereby causing borgmatic to error on them, use the
+following borgmatic configuration with Borg 1.2 or below:
+
+```yaml
+borg_exit_codes:
+    - code: 1
+      treat_as: error
+```
+
+Be aware though that Borg exits with a warning code for a variety of benign
+situations such as files changing while they're being read, so this example
+may not meet your needs.
+
+Here's another Borg 1.2 example that squashes Borg errors to warnings:
+
+```yaml
+borg_exit_codes:
+    - code: 2
+      treat_as: warning
+```
+
+Be careful with this example though, because it prevents borgmatic from
+erroring when Borg errors, which may not be desirable.

docs/how-to/make-per-application-backups.md

@@ -141,7 +141,7 @@ for more information.
 
 <span class="minilink minilink-addedin">With Borg version 2.x</span>See the
 [match archives
-documentation](https://borgbackup.readthedocs.io/en/2.0.0b13/usage/help.html#borg-help-match-archives).
+documentation](https://borgbackup.readthedocs.io/en/2.0.0b16/usage/help.html#borg-help-match-archives).
 
 Some borgmatic command-line actions also have a `--match-archives` flag that
 overrides both the auto-matching behavior and the `match_archives`

docs/how-to/upgrade.md

@@ -193,7 +193,7 @@ confirmation of success—or tells you if something hasn't been transferred yet.
 Note that by omitting the `--upgrader` flag, you can also do archive transfers
 between related Borg 2 repositories without upgrading, even down to individual
 archives. For more on that functionality, see the [Borg transfer
-documentation](https://borgbackup.readthedocs.io/en/2.0.0b13/usage/transfer.html).
+documentation](https://borgbackup.readthedocs.io/en/2.0.0b16/usage/transfer.html).
 
 That's it! Now you can use your new Borg 2 repository as normal with
 borgmatic. If you've got multiple repositories, repeat the above process for

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "2.0.5"
+version = "2.0.6"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]

tests/integration/config/test_generate.py

@@ -126,6 +126,45 @@ def test_schema_to_sample_configuration_comments_out_non_source_config_options_i
     assert 'field3' not in config[0].ca.items
 
 
+def test_schema_to_sample_configuration_comments_out_non_source_config_options_in_sequence_of_maps_with_different_subschemas():
+    schema = {
+        'type': 'array',
+        'items': {
+            'type': 'object',
+            'oneOf': [
+                {
+                    'properties': dict(
+                        [
+                            ('field1', {'type': 'string', 'example': 'Example 1'}),
+                            ('field2', {'type': 'string', 'example': 'Example 2'}),
+                        ]
+                    )
+                },
+                {
+                    'properties': dict(
+                        [
+                            ('field2', {'type': 'string', 'example': 'Example 2'}),
+                            ('field3', {'type': 'string', 'example': 'Example 3'}),
+                        ]
+                    )
+                },
+            ],
+        },
+    }
+    source_config = [{'field1': 'value'}, {'field3': 'value'}]
+
+    config = module.schema_to_sample_configuration(schema, source_config)
+
+    assert config == [
+        dict([('field1', 'Example 1'), ('field2', 'Example 2'), ('field3', 'Example 3')])
+    ]
+
+    # The first field in a sequence does not get commented.
+    assert 'field1' not in config[0].ca.items
+    assert 'COMMENT_OUT' in config[0].ca.items['field2'][1][-1]._value
+    assert 'COMMENT_OUT' in config[0].ca.items['field3'][1][-1]._value
+
+
 def test_comment_out_line_skips_blank_line():
     line = '    \n'
 
@@ -169,8 +208,11 @@ foo:
         - quux
 
 repositories:
-    - one
-    - two
+    - path: foo
+      # COMMENT_OUT
+      label: bar
+    - path: baz
+      label: quux
 
 # This comment should be kept.
 # COMMENT_OUT
@@ -185,8 +227,10 @@ other: thing
 #         - quux
 
 repositories:
-    - one
-    - two
+    - path: foo
+#       label: bar
+    - path: baz
+      label: quux
 
 # This comment should be kept.
 # other: thing
@@ -297,14 +341,28 @@ def test_add_comments_to_configuration_comments_out_non_source_config_options():
     assert 'baz' not in config.ca.items
 
 
-def test_add_comments_to_configuration_object_with_skip_first_does_not_comment_out_first_option():
+def test_add_comments_to_configuration_object_with_skip_first_field_does_not_comment_out_first_option():
     config = module.ruamel.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
     schema = {
         'type': 'object',
         'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}},
     }
 
-    module.add_comments_to_configuration_object(config, schema, skip_first=True)
+    module.add_comments_to_configuration_object(config, schema, skip_first_field=True)
+
+    assert 'foo' not in config.ca.items
+    assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value
+    assert 'baz' not in config.ca.items
+
+
+def test_add_comments_to_configuration_object_with_skip_first_field_does_not_comment_out_first_option():
+    config = module.ruamel.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
+    schema = {
+        'type': 'object',
+        'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}},
+    }
+
+    module.add_comments_to_configuration_object(config, schema, skip_first_field=True)
 
     assert 'foo' not in config.ca.items
     assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value
@@ -326,15 +384,39 @@ def test_generate_sample_configuration_does_not_raise():
     module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml')
 
 
-def test_generate_sample_configuration_with_source_filename_does_not_raise():
+def test_generate_sample_configuration_with_source_filename_omits_empty_bootstrap_field():
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('schema.yaml').and_return('')
     flexmock(module.ruamel.yaml).should_receive('YAML').and_return(
         flexmock(load=lambda filename: {})
     )
-    flexmock(module.load).should_receive('load_configuration')
+    flexmock(module.load).should_receive('load_configuration').and_return(
+        {'bootstrap': {}, 'foo': 'bar'}
+    )
     flexmock(module.normalize).should_receive('normalize')
-    flexmock(module).should_receive('schema_to_sample_configuration')
+    flexmock(module).should_receive('schema_to_sample_configuration').with_args(
+        object, {'foo': 'bar'}
+    ).once()
+    flexmock(module).should_receive('merge_source_configuration_into_destination')
+    flexmock(module).should_receive('render_configuration')
+    flexmock(module).should_receive('comment_out_optional_configuration')
+    flexmock(module).should_receive('write_configuration')
+
+    module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml')
+
+
+def test_generate_sample_configuration_with_source_filename_keeps_non_empty_bootstrap_field():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module.ruamel.yaml).should_receive('YAML').and_return(
+        flexmock(load=lambda filename: {})
+    )
+    source_config = {'bootstrap': {'stuff': 'here'}, 'foo': 'bar'}
+    flexmock(module.load).should_receive('load_configuration').and_return(source_config)
+    flexmock(module.normalize).should_receive('normalize')
+    flexmock(module).should_receive('schema_to_sample_configuration').with_args(
+        object, source_config
+    ).once()
     flexmock(module).should_receive('merge_source_configuration_into_destination')
     flexmock(module).should_receive('render_configuration')
     flexmock(module).should_receive('comment_out_optional_configuration')

tests/unit/actions/test_check.py

@@ -601,6 +601,53 @@ def test_collect_spot_check_source_paths_parses_borg_output():
     ) == ('/etc/path', '/etc/other')
 
 
+def test_collect_spot_check_source_paths_omits_progress_from_create_dry_run_command():
+    flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
+        {'hook1': False, 'hook2': False}
+    )
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
+        [Pattern('foo'), Pattern('bar')]
+    )
+    flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
+        dry_run=True,
+        repository_path='repo',
+        config={'working_directory': '/', 'list_details': True},
+        patterns=[Pattern('foo'), Pattern('bar')],
+        local_borg_version=object,
+        global_arguments=object,
+        borgmatic_runtime_directory='/run/borgmatic',
+        local_path=object,
+        remote_path=object,
+        stream_processes=False,
+    ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
+    flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).and_return(
+        'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
+    )
+    flexmock(module.os.path).should_receive('isfile').and_return(True)
+
+    assert module.collect_spot_check_source_paths(
+        repository={'path': 'repo'},
+        config={'working_directory': '/', 'progress': True},
+        local_borg_version=flexmock(),
+        global_arguments=flexmock(),
+        local_path=flexmock(),
+        remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    ) == ('/etc/path', '/etc/other')
+
+
 def test_collect_spot_check_source_paths_passes_through_stream_processes_false():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': False}
@@ -1024,7 +1071,9 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
     flexmock(module.os.path).should_receive('islink').and_return(False)
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
-    ).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
+    ).with_args(
+        ('/usr/local/bin/xxhsum', '-H64', '/foo', '/bar'), working_directory=None
+    ).and_return(
         'hash1  /foo\nhash2  /bar'
     )
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
@@ -1039,7 +1088,7 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
                 {
                     'name': 'spot',
                     'data_sample_percentage': 50,
-                    'xxh64sum_command': '/usr/local/bin/xxh64sum',
+                    'xxh64sum_command': '/usr/local/bin/xxhsum -H64',
                 },
             ]
         },

tests/unit/config/test_normalize.py

@@ -334,6 +334,16 @@ def test_normalize_commands_moves_individual_command_hooks_to_unified_commands(
             {'repositories': [{'path': 'rclone:host:repo'}]},
             True,
         ),
+        (
+            {'repositories': ['s3:stuff']},
+            {'repositories': [{'path': 's3:stuff'}]},
+            True,
+        ),
+        (
+            {'repositories': ['b2:stuff']},
+            {'repositories': [{'path': 'b2:stuff'}]},
+            True,
+        ),
         (
             {'repositories': ['file:///repo']},
             {'repositories': [{'path': '/repo'}]},

tests/unit/hooks/data_source/test_lvm.py

@@ -214,6 +214,7 @@ def test_snapshot_logical_volume_with_percentage_snapshot_name_uses_lvcreate_ext
             '/dev/snap',
         ),
         output_log_level=object,
+        close_fds=True,
     )
 
     module.snapshot_logical_volume('lvcreate', 'snap', '/dev/snap', '10%ORIGIN')
@@ -233,6 +234,7 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
             '/dev/snap',
         ),
         output_log_level=object,
+        close_fds=True,
     )
 
     module.snapshot_logical_volume('lvcreate', 'snap', '/dev/snap', '10TB')
@@ -554,16 +556,7 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte
     )
     flexmock(module.os).should_receive('getpid').and_return(1234)
     flexmock(module).should_receive('snapshot_logical_volume').never()
-    flexmock(module).should_receive('get_snapshots').with_args(
-        'lvs', snapshot_name='lvolume1_borgmatic-1234'
-    ).and_return(
-        (module.Snapshot(name='lvolume1_borgmatic-1234', device_path='/dev/lvolume1_snap'),)
-    )
-    flexmock(module).should_receive('get_snapshots').with_args(
-        'lvs', snapshot_name='lvolume2_borgmatic-1234'
-    ).and_return(
-        (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),)
-    )
+    flexmock(module).should_receive('get_snapshots').never()
     flexmock(module).should_receive('mount_snapshot').never()
 
     assert (