Attempt to repair any inconsistencies found during a consistency check via "borgmatic check --repair" flag (#266).

NEWS

@@ -1,3 +1,10 @@
+1.4.17
+ * #235: Pass extra options directly to particular Borg commands, handy for Borg options that
+   borgmatic does not yet support natively. Use "extra_borg_options" in the storage configuration
+   section.
+ * #266: Attempt to repair any inconsistencies found during a consistency check via
+   "borgmatic check --repair" flag.
+
 1.4.16
  * #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and
    has an exit code of 1.

borgmatic/borg/check.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import extract
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_without_capture
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
@@ -91,23 +91,23 @@ def check_archives(
     consistency_config,
     local_path='borg',
     remote_path=None,
+    repair=None,
     only_checks=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, a consistency config dict,
-    local/remote commands to run, and an optional list of checks to use instead of configured
-    checks, check the contained Borg archives for consistency.
+    local/remote commands to run, whether to attempt a repair, and an optional list of checks
+    to use instead of configured checks, check the contained Borg archives for consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config, only_checks)
     check_last = consistency_config.get('check_last', None)
     lock_wait = None
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
 
     if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
-        remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
         lock_wait = storage_config.get('lock_wait', None)
-        lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
 
         verbosity_flags = ()
         if logger.isEnabledFor(logging.INFO):
@@ -119,13 +119,21 @@ def check_archives(
 
         full_command = (
             (local_path, 'check')
+            + (('--repair',) if repair else ())
             + _make_check_flags(checks, check_last, prefix)
-            + remote_path_flags
-            + lock_wait_flags
+            + (('--remote-path', remote_path) if remote_path else ())
+            + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
+            + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
             + (repository,)
         )
 
+        # The Borg repair option trigger an interactive prompt, which won't work when output is
+        # captured.
+        if repair:
+            execute_command_without_capture(full_command, error_on_warnings=True)
+            return
+
         execute_command(full_command, error_on_warnings=True)
 
     if 'extract' in checks:

borgmatic/borg/create.py

@@ -150,6 +150,7 @@ def create_archive(
     files_cache = location_config.get('files_cache')
     default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
     archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
 
     full_command = (
         (local_path, 'create')
@@ -185,6 +186,7 @@ def create_archive(
         + (('--dry-run',) if dry_run else ())
         + (('--progress',) if progress else ())
         + (('--json',) if json else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + (
             '{repository}::{archive_name_format}'.format(
                 repository=repository, archive_name_format=archive_name_format

borgmatic/borg/init.py

@@ -11,6 +11,7 @@ INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
 
 def initialize_repository(
     repository,
+    storage_config,
     encryption_mode,
     append_only=None,
     storage_quota=None,
@@ -18,9 +19,9 @@ def initialize_repository(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a Borg encryption mode, whether the repository should
-    be append-only, and the storage quota to use, initialize the repository. If the repository
-    already exists, then log and skip initialization.
+    Given a local or remote repository path, a storage configuration dict, a Borg encryption mode,
+    whether the repository should be append-only, and the storage quota to use, initialize the
+    repository. If the repository already exists, then log and skip initialization.
     '''
     info_command = (local_path, 'info', repository)
     logger.debug(' '.join(info_command))
@@ -33,6 +34,8 @@ def initialize_repository(
         if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
             raise
 
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('init', '')
+
     init_command = (
         (local_path, 'init')
         + (('--encryption', encryption_mode) if encryption_mode else ())
@@ -41,6 +44,7 @@ def initialize_repository(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--remote-path', remote_path) if remote_path else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + (repository,)
     )
 

borgmatic/borg/prune.py

@@ -49,6 +49,7 @@ def prune_archives(
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '')
 
     full_command = (
         (local_path, 'prune')
@@ -61,6 +62,7 @@ def prune_archives(
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--stats',) if stats else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + (repository,)
     )
 

borgmatic/commands/arguments.py

@@ -266,6 +266,13 @@ def parse_arguments(*unparsed_arguments):
         add_help=False,
     )
     check_group = check_parser.add_argument_group('check arguments')
+    check_group.add_argument(
+        '--repair',
+        dest='repair',
+        default=False,
+        action='store_true',
+        help='Attempt to repair any inconsistencies found (experimental and only for interactive use)',
+    )
     check_group.add_argument(
         '--only',
         metavar='CHECK',

borgmatic/commands/borgmatic.py

@@ -189,6 +189,7 @@ def run_actions(
         logger.info('{}: Initializing repository'.format(repository))
         borg_init.initialize_repository(
             repository,
+            storage,
             arguments['init'].encryption_mode,
             arguments['init'].append_only,
             arguments['init'].storage_quota,
@@ -229,6 +230,7 @@ def run_actions(
             consistency,
             local_path=local_path,
             remote_path=remote_path,
+            repair=arguments['check'].repair,
             only_checks=arguments['check'].only,
         )
     if 'extract' in arguments:

borgmatic/config/schema.yaml

@@ -245,6 +245,29 @@ map:
                     Bypass Borg error about a previously unknown unencrypted repository. Defaults to
                     false.
                 example: true
+            extra_borg_options:
+                map:
+                    init:
+                        type: str
+                        desc: Extra command-line options to pass to "borg init".
+                        example: "--make-parent-dirs"
+                    prune:
+                        type: str
+                        desc: Extra command-line options to pass to "borg prune".
+                        example: "--save-space"
+                    create:
+                        type: str
+                        desc: Extra command-line options to pass to "borg create".
+                        example: "--no-files-cache"
+                    check:
+                        type: str
+                        desc: Extra command-line options to pass to "borg check".
+                        example: "--save-space"
+                desc: |
+                    Additional options to pass directly to particular Borg commands, handy for Borg
+                    options that borgmatic does not yet support natively. Note that borgmatic does
+                    not perform any validation on these options. Running borgmatic with
+                    "--verbosity 2" shows the exact Borg command-line invocation.
     retention:
         desc: |
             Retention policy for how many backups to keep in each category. See

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.4.16'
+VERSION = '1.4.17'
 
 
 setup(

tests/unit/borg/test_check.py

@@ -158,6 +158,21 @@ def test_make_check_flags_with_default_checks_and_prefix_includes_prefix_flag():
     assert flags == ('--prefix', 'foo-')
 
 
+def test_check_archives_with_repair_calls_borg_with_repair_parameter():
+    checks = ('repository',)
+    consistency_config = {'check_last': None}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    flexmock(module).should_receive('execute_command').never()
+    flexmock(module).should_receive('execute_command_without_capture').with_args(
+        ('borg', 'check', '--repair', 'repo'), error_on_warnings=True
+    ).once()
+
+    module.check_archives(
+        repository='repo', storage_config={}, consistency_config=consistency_config, repair=True
+    )
+
+
 @pytest.mark.parametrize(
     'checks',
     (
@@ -296,3 +311,17 @@ def test_check_archives_with_retention_prefix():
     module.check_archives(
         repository='repo', storage_config={}, consistency_config=consistency_config
     )
+
+
+def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options():
+    checks = ('repository',)
+    consistency_config = {'check_last': None}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo'))
+
+    module.check_archives(
+        repository='repo',
+        storage_config={'extra_borg_options': {'check': '--extra --options'}},
+        consistency_config=consistency_config,
+    )

tests/unit/borg/test_create.py

@@ -1092,3 +1092,28 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
         },
         storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},
     )
+
+
+def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        error_on_warnings=False,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'extra_borg_options': {'create': '--extra --options'}},
+    )

tests/unit/borg/test_init.py

@@ -32,7 +32,7 @@ def test_initialize_repository_calls_borg_with_parameters():
     insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('repo',))
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey')
+    module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
 
 
 def test_initialize_repository_raises_for_borg_init_error():
@@ -42,14 +42,16 @@ def test_initialize_repository_raises_for_borg_init_error():
     )
 
     with pytest.raises(subprocess.CalledProcessError):
-        module.initialize_repository(repository='repo', encryption_mode='repokey')
+        module.initialize_repository(
+            repository='repo', storage_config={}, encryption_mode='repokey'
+        )
 
 
 def test_initialize_repository_skips_initialization_when_repository_already_exists():
     insert_info_command_found_mock()
     flexmock(module).should_receive('execute_command_without_capture').never()
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey')
+    module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
 
 
 def test_initialize_repository_raises_for_unknown_info_command_error():
@@ -58,21 +60,27 @@ def test_initialize_repository_raises_for_unknown_info_command_error():
     )
 
     with pytest.raises(subprocess.CalledProcessError):
-        module.initialize_repository(repository='repo', encryption_mode='repokey')
+        module.initialize_repository(
+            repository='repo', storage_config={}, encryption_mode='repokey'
+        )
 
 
 def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter():
     insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--append-only', 'repo'))
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey', append_only=True)
+    module.initialize_repository(
+        repository='repo', storage_config={}, encryption_mode='repokey', append_only=True
+    )
 
 
 def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter():
     insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G', 'repo'))
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey', storage_quota='5G')
+    module.initialize_repository(
+        repository='repo', storage_config={}, encryption_mode='repokey', storage_quota='5G'
+    )
 
 
 def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
@@ -80,7 +88,7 @@ def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
     insert_init_command_mock(INIT_COMMAND + ('--info', 'repo'))
     insert_logging_mock(logging.INFO)
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey')
+    module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
 
 
 def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
@@ -88,18 +96,33 @@ def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
     insert_init_command_mock(INIT_COMMAND + ('--debug', 'repo'))
     insert_logging_mock(logging.DEBUG)
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey')
+    module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
 
 
 def test_initialize_repository_with_local_path_calls_borg_via_local_path():
     insert_info_command_not_found_mock()
     insert_init_command_mock(('borg1',) + INIT_COMMAND[1:] + ('repo',))
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey', local_path='borg1')
+    module.initialize_repository(
+        repository='repo', storage_config={}, encryption_mode='repokey', local_path='borg1'
+    )
 
 
 def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter():
     insert_info_command_not_found_mock()
     insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1', 'repo'))
 
-    module.initialize_repository(repository='repo', encryption_mode='repokey', remote_path='borg1')
+    module.initialize_repository(
+        repository='repo', storage_config={}, encryption_mode='repokey', remote_path='borg1'
+    )
+
+
+def test_initialize_repository_with_extra_borg_options_calls_borg_with_extra_options():
+    insert_info_command_not_found_mock()
+    insert_init_command_mock(INIT_COMMAND + ('--extra', '--options', 'repo'))
+
+    module.initialize_repository(
+        repository='repo',
+        storage_config={'extra_borg_options': {'init': '--extra --options'}},
+        encryption_mode='repokey',
+    )

tests/unit/borg/test_prune.py

@@ -188,3 +188,18 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
         storage_config=storage_config,
         retention_config=retention_config,
     )
+
+
+def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS
+    )
+    insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO)
+
+    module.prune_archives(
+        dry_run=False,
+        repository='repo',
+        storage_config={'extra_borg_options': {'prune': '--extra --options'}},
+        retention_config=retention_config,
+    )