Override configured consistency checks via "borgmatic check --only" command-line flag (#210).

https://yaml.org/spec/1.2/spec.html#id2780069

NEWS

@@ -1,3 +1,9 @@
+1.3.16
+ * #210: Support for Borg check --verify-data flag via borgmatic "data" consistency check.
+ * #210: Override configured consistency checks via "borgmatic check --only" command-line flag.
+ * When generating sample configuration with generate-borgmatic-config, add a space after each "#"
+   comment indicator.
+
 1.3.15
  * #208: Fix for traceback when the "checks" option has an empty value.
  * #209: Bypass Borg error about a moved repository via "relocated_repo_access_is_ok" option in

borgmatic/borg/check.py

@@ -10,9 +10,10 @@ DEFAULT_PREFIX = '{hostname}-'
 logger = logging.getLogger(__name__)
 
 
-def _parse_checks(consistency_config):
+def _parse_checks(consistency_config, only_checks=None):
     '''
-    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
+    Given a consistency config with a "checks" list, and an optional list of override checks,
+    transform them a tuple of named checks to run.
 
     For example, given a retention config of:
 
@@ -22,16 +23,21 @@ def _parse_checks(consistency_config):
 
         ('repository', 'archives')
 
-    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
-    "disabled", return an empty tuple, meaning that no checks should be run.
+    If no "checks" option is present in the config, return the DEFAULT_CHECKS. If the checks value
+    is the string "disabled", return an empty tuple, meaning that no checks should be run.
+
+    If the "data" option is present, then make sure the "archives" option is included as well.
     '''
-    checks = consistency_config.get('checks', []) or []
+    checks = [
+        check.lower() for check in (only_checks or consistency_config.get('checks', []) or [])
+    ]
     if checks == ['disabled']:
         return ()
 
-    return (
-        tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
-    )
+    if 'data' in checks and 'archives' not in checks:
+        checks.append('archives')
+
+    return tuple(check for check in checks if check not in ('disabled', '')) or DEFAULT_CHECKS
 
 
 def _make_check_flags(checks, check_last=None, prefix=None):
@@ -68,30 +74,37 @@ def _make_check_flags(checks, check_last=None, prefix=None):
                 'Ignoring consistency prefix option, as "archives" is not in consistency checks.'
             )
 
+    common_flags = last_flags + prefix_flags + (('--verify-data',) if 'data' in checks else ())
+
     if set(DEFAULT_CHECKS).issubset(set(checks)):
-        return last_flags + prefix_flags
+        return common_flags
 
     return (
         tuple('--{}-only'.format(check) for check in checks if check in DEFAULT_CHECKS)
-        + last_flags
-        + prefix_flags
+        + common_flags
     )
 
 
 def check_archives(
-    repository, storage_config, consistency_config, local_path='borg', remote_path=None
+    repository,
+    storage_config,
+    consistency_config,
+    local_path='borg',
+    remote_path=None,
+    only_checks=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, a consistency config dict,
-    and a local/remote commands to run, check the contained Borg archives for consistency.
+    local/remote commands to run, and an optional list of checks to use instead of configured
+    checks, check the contained Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
-    checks = _parse_checks(consistency_config)
+    checks = _parse_checks(consistency_config, only_checks)
     check_last = consistency_config.get('check_last', None)
     lock_wait = None
 
-    if set(checks).intersection(set(DEFAULT_CHECKS)):
+    if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
         remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
         lock_wait = storage_config.get('lock_wait', None)
         lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()

borgmatic/commands/arguments.py

@@ -212,6 +212,14 @@ def parse_arguments(*unparsed_arguments):
         add_help=False,
     )
     check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--only',
+        metavar='CHECK',
+        choices=('repository', 'archives', 'data', 'extract'),
+        dest='only',
+        action='append',
+        help='Run a particular consistency check instead of configured checks (can specify multiple times)',
+    )
     check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
     extract_parser = subparsers.add_parser(

borgmatic/commands/borgmatic.py

@@ -147,7 +147,12 @@ def run_actions(
     if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
-            repository, storage, consistency, local_path=local_path, remote_path=remote_path
+            repository,
+            storage,
+            consistency,
+            local_path=local_path,
+            remote_path=remote_path,
+            only_checks=arguments['check'].only,
         )
     if 'extract' in arguments:
         if arguments['extract'].repository is None or repository == arguments['extract'].repository:

borgmatic/config/generate.py

@@ -45,10 +45,10 @@ def _comment_out_line(line):
     # Comment out the names of optional sections.
     one_indent = ' ' * INDENT
     if not line.startswith(one_indent):
-        return '#' + line
+        return '# ' + line
 
     # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
-    return '#'.join((one_indent, line[INDENT:]))
+    return '# '.join((one_indent, line[INDENT:]))
 
 
 REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}

borgmatic/config/schema.yaml

@@ -290,14 +290,15 @@ map:
             checks:
                 seq:
                     - type: str
-                      enum: ['repository', 'archives', 'extract', 'disabled']
+                      enum: ['repository', 'archives', 'data', 'extract', 'disabled']
                       unique: true
                 desc: |
-                    List of one or more consistency checks to run: "repository", "archives", and/or
-                    "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
-                    all consistency checks. "repository" checks the consistency of the repository,
-                    "archive" checks all of the archives, and "extract" does an extraction dry-run
-                    of the most recent archive.
+                    List of one or more consistency checks to run: "repository", "archives", "data",
+                    and/or "extract". Defaults to "repository" and "archives". Set to "disabled" to
+                    disable all consistency checks. "repository" checks the consistency of the
+                    repository, "archives" checks all of the archives, "data" verifies the integrity
+                    of the data within the archives, and "extract" does an extraction dry-run of the
+                    most recent archive. Note that "data" implies "archives".
                 example:
                     - repository
                     - archives

docs/how-to/set-up-backups.md

@@ -4,7 +4,7 @@ title: How to set up backups with borgmatic
 ## Installation
 
 To get up and running, first [install
-Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
+Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at
 least version 1.1.
 
 Borgmatic consumes configurations in `/etc/borgmatic/` and `/etc/borgmatic.d/`
@@ -88,7 +88,7 @@ encrypt your Borg repository with a passphrase instead of a key file, you'll
 either need to set the borgmatic `encryption_passphrase` configuration
 variable or set the `BORG_PASSPHRASE` environment variable. See the
 [repository encryption
-section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
+section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption)
 of the Borg Quick Start for more info.
 
 Alternatively, you can specify the passphrase programatically by setting
@@ -133,11 +133,11 @@ This uses the borgmatic configuration file you created above to determine
 which local or remote repository to create, and encrypts it with the
 encryption passphrase specified there if one is provided. Read about [Borg
 encryption
-modes](https://borgbackup.readthedocs.io/en/latest/usage/init.html#encryption-modes)
+modes](https://borgbackup.readthedocs.io/en/stable/usage/init.html#encryption-modes)
 for the menu of available encryption modes.
 
 Also, optionally check out the [Borg Quick
-Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) for more
+Start](https://borgbackup.readthedocs.org/en/stable/quickstart.html) for more
 background about repository initialization.
 
 Note that borgmatic skips repository initialization if the repository already

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.3.15'
+VERSION = '1.3.16'
 
 
 setup(

tests/integration/config/test_generate.py

@@ -31,17 +31,17 @@ def test_comment_out_line_skips_already_commented_out_line():
 def test_comment_out_line_comments_section_name():
     line = 'figgy-pudding:'
 
-    assert module._comment_out_line(line) == '#' + line
+    assert module._comment_out_line(line) == '# ' + line
 
 
 def test_comment_out_line_comments_indented_option():
     line = '    enabled: true'
 
-    assert module._comment_out_line(line) == '    #enabled: true'
+    assert module._comment_out_line(line) == '    # enabled: true'
 
 
 def test_comment_out_optional_configuration_comments_optional_config_only():
-    flexmock(module)._comment_out_line = lambda line: '#' + line
+    flexmock(module)._comment_out_line = lambda line: '# ' + line
     config = '''
 foo:
     bar:
@@ -56,18 +56,19 @@ location:
     other: thing
     '''
 
+    # flake8: noqa
     expected_config = '''
-#foo:
-#    bar:
-#        - baz
-#        - quux
-#
+# foo:
+#     bar:
+#         - baz
+#         - quux
+# 
 location:
     repositories:
         - one
         - two
-#
-#    other: thing
+# 
+#     other: thing
     '''
 
     assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()

tests/unit/borg/test_check.py

@@ -46,12 +46,48 @@ def test_parse_checks_with_disabled_returns_no_checks():
     assert checks == ()
 
 
+def test_parse_checks_with_data_check_also_injects_archives():
+    checks = module._parse_checks({'checks': ['data']})
+
+    assert checks == ('data', 'archives')
+
+
+def test_parse_checks_with_data_check_passes_through_archives():
+    checks = module._parse_checks({'checks': ['data', 'archives']})
+
+    assert checks == ('data', 'archives')
+
+
+def test_parse_checks_prefers_override_checks_to_configured_checks():
+    checks = module._parse_checks({'checks': ['archives']}, only_checks=['repository', 'extract'])
+
+    assert checks == ('repository', 'extract')
+
+
+def test_parse_checks_with_override_data_check_also_injects_archives():
+    checks = module._parse_checks({'checks': ['extract']}, only_checks=['data'])
+
+    assert checks == ('data', 'archives')
+
+
 def test_make_check_flags_with_repository_check_returns_flag():
     flags = module._make_check_flags(('repository',))
 
     assert flags == ('--repository-only',)
 
 
+def test_make_check_flags_with_archives_check_returns_flag():
+    flags = module._make_check_flags(('archives',))
+
+    assert flags == ('--archives-only',)
+
+
+def test_make_check_flags_with_data_check_returns_flag():
+    flags = module._make_check_flags(('data',))
+
+    assert flags == ('--verify-data',)
+
+
 def test_make_check_flags_with_extract_omits_extract_flag():
     flags = module._make_check_flags(('extract',))