Support for Borg --lock-wait option for the maximum wait for a repository/cache lock (#56).

AUTHORS

@@ -4,6 +4,7 @@ Alexander Görtz: Python 3 compatibility
 Henning Schroeder: Copy editing
 Johannes Feichtner: Support for user hooks
 Michele Lazzeri: Custom archive names
+newtonne: Read encryption password from external file
 Robin `ypid` Schneider: Support additional options of Borg
 Scott Squires: Custom archive names
 Thomas LÉVEIL: Support for a keep_minutely prune option

NEWS

@@ -1,3 +1,15 @@
+1.1.15
+ * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
+ * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
+   Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
+ * #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
+ * #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
+ * #58: Support for using tilde in exclude_patterns to reference home directory.
+
+1.1.14
+ * #49: Fix for typo in --patterns-from option.
+ * #47: Support for Borg --dry-run option via borgmatic command-line.
+
 1.1.13
  * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
    "archives", and "extract") are specified in borgmatic configuration.

README.md

@@ -57,6 +57,12 @@ environment variable. See the [repository encryption
 section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
 of the Quick Start for more info.
 
+Alternatively, the passphrase can be specified programatically by setting
+either the borgmatic `encryption_passcommand` configuration variable or the
+`BORG_PASSCOMMAND` environment variable. See the [Borg Security
+FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-encryption-passphrase-programmatically)
+for more info.
+
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.
 

borgmatic/borg/check.py

@@ -60,18 +60,23 @@ def _make_check_flags(checks, check_last=None):
     ) + last_flag
 
 
-def check_archives(verbosity, repository, consistency_config, local_path='borg', remote_path=None):
+def check_archives(verbosity, repository, storage_config, consistency_config, local_path='borg',
+                   remote_path=None):
     '''
-    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    local/remote commands to run, check the contained Borg archives for consistency.
+    Given a verbosity flag, a local or remote repository path, a storage config dict, a consistency
+    config dict, and a local/remote commands to run, check the contained Borg archives for
+    consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config)
     check_last = consistency_config.get('check_last', None)
+    lock_wait = None
 
     if set(checks).intersection(set(DEFAULT_CHECKS)):
         remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+        lock_wait = storage_config.get('lock_wait', None)
+        lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
         verbosity_flags = {
             VERBOSITY_SOME: ('--info',),
             VERBOSITY_LOTS: ('--debug',),
@@ -80,7 +85,7 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
         full_command = (
             local_path, 'check',
             repository,
-        ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
+        ) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
 
         # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
         stdout = None if verbosity_flags else open(os.devnull, 'w')
@@ -89,4 +94,4 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
         subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
 
     if 'extract' in checks:
-        extract.extract_last_archive_dry_run(verbosity, repository, local_path, remote_path)
+        extract.extract_last_archive_dry_run(verbosity, repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -12,6 +12,10 @@ logger = logging.getLogger(__name__)
 
 
 def initialize_environment(storage_config):
+    passcommand = storage_config.get('encryption_passcommand')
+    if passcommand:
+        os.environ['BORG_PASSCOMMAND'] = passcommand
+
     passphrase = storage_config.get('encryption_passphrase')
     if passphrase:
         os.environ['BORG_PASSPHRASE'] = passphrase
@@ -31,6 +35,22 @@ def _expand_directory(directory):
     return glob.glob(expanded_directory) or [expanded_directory]
 
 
+def _expand_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(
+            _expand_directory(directory)
+            for directory in directories
+        )
+    )
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -57,7 +77,7 @@ def _make_pattern_flags(location_config, pattern_filename=None):
 
     return tuple(
         itertools.chain.from_iterable(
-            ('--pattern-from', pattern_filename)
+            ('--patterns-from', pattern_filename)
             for pattern_filename in pattern_filenames
         )
     )
@@ -85,25 +105,20 @@ def _make_exclude_flags(location_config, exclude_filename=None):
 
 
 def create_archive(
-    verbosity, repository, location_config, storage_config, local_path='borg', remote_path=None,
+    verbosity, dry_run, repository, location_config, storage_config, local_path='borg', remote_path=None,
 ):
     '''
-    Given a vebosity flag, a local or remote repository path, a location config dict, and a storage
-    config dict, create a Borg archive.
+    Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
+    storage config dict, create a Borg archive.
     '''
-    sources = tuple(
-        itertools.chain.from_iterable(
-            _expand_directory(directory)
-            for directory in location_config['source_directories']
-        )
-    )
+    sources = _expand_directories(location_config['source_directories'])
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
     pattern_flags = _make_pattern_flags(
         location_config,
         pattern_file.name if pattern_file else None,
     )
-    exclude_file = _write_pattern_file(location_config.get('exclude_patterns'))
+    exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
     exclude_flags = _make_exclude_flags(
         location_config,
         exclude_file.name if exclude_file else None,
@@ -114,14 +129,17 @@ def create_archive(
     remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
     umask = storage_config.get('umask', None)
     umask_flags = ('--umask', str(umask)) if umask else ()
+    lock_wait = storage_config.get('lock_wait', None)
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
     files_cache = location_config.get('files_cache')
     files_cache_flags = ('--files-cache', files_cache) if files_cache else ()
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
-        VERBOSITY_SOME: ('--info', '--stats',),
-        VERBOSITY_LOTS: ('--debug', '--list', '--stats'),
+        VERBOSITY_SOME: ('--info',) if dry_run else ('--info', '--stats',),
+        VERBOSITY_LOTS: ('--debug', '--list',) if dry_run else ('--debug', '--list', '--stats',),
     }.get(verbosity, ())
+    dry_run_flags = ('--dry-run',) if dry_run else ()
     default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
     archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
 
@@ -133,7 +151,7 @@ def create_archive(
         ),
     ) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
         one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
-        verbosity_flags
+        lock_wait_flags + verbosity_flags + dry_run_flags
 
     logger.debug(' '.join(full_command))
     subprocess.check_call(full_command)

borgmatic/borg/extract.py

@@ -8,12 +8,13 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 logger = logging.getLogger(__name__)
 
 
-def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remote_path=None):
+def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_path='borg', remote_path=None):
     '''
     Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
     the dry-run.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info',),
         VERBOSITY_LOTS: ('--debug',),
@@ -23,7 +24,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
         local_path, 'list',
         '--short',
         repository,
-    ) + remote_path_flags + verbosity_flags
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags
 
     list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
 
@@ -39,7 +40,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
             repository=repository,
             last_archive_name=last_archive_name,
         ),
-    ) + remote_path_flags + verbosity_flags + list_flag
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag
 
     logger.debug(' '.join(full_extract_command))
     subprocess.check_call(full_extract_command)

borgmatic/borg/prune.py

@@ -32,16 +32,21 @@ def _make_prune_flags(retention_config):
     )
 
 
-def prune_archives(verbosity, repository, retention_config, local_path='borg', remote_path=None):
+def prune_archives(verbosity, dry_run, repository, storage_config, retention_config,
+                   local_path='borg', remote_path=None):
     '''
-    Given a verbosity flag, a local or remote repository path, a retention config dict, prune Borg
-    archives according the the retention policy specified in that configuration.
+    Given verbosity/dry-run flags, a local or remote repository path, a storage config dict, and a
+    retention config dict, prune Borg archives according the the retention policy specified in that
+    configuration.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    lock_wait = storage_config.get('lock_wait', None)
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info', '--stats',),
         VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
     }.get(verbosity, ())
+    dry_run_flags = ('--dry-run',) if dry_run else ()
 
     full_command = (
         local_path, 'prune',
@@ -50,7 +55,7 @@ def prune_archives(verbosity, repository, retention_config, local_path='borg', r
         element
         for pair in _make_prune_flags(retention_config)
         for element in pair
-    ) + remote_path_flags + verbosity_flags
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags + dry_run_flags
 
     logger.debug(' '.join(full_command))
     subprocess.check_call(full_command)

borgmatic/commands/borgmatic.py

@@ -61,6 +61,12 @@ def parse_arguments(*arguments):
         action='store_true',
         help='Check archives for consistency',
     )
+    parser.add_argument(
+        '-n', '--dry-run',
+        dest='dry_run',
+        action='store_true',
+        help='Go through the motions, but do not actually write any changes to the repository',
+    )
     parser.add_argument(
         '-v', '--verbosity',
         type=int,
@@ -100,19 +106,23 @@ def run_configuration(config_filename, args):  # pragma: no cover
 
         for unexpanded_repository in location['repositories']:
             repository = os.path.expanduser(unexpanded_repository)
+            dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
             if args.prune:
-                logger.info('{}: Pruning archives'.format(repository))
+                logger.info('{}: Pruning archives{}'.format(repository, dry_run_label))
                 prune.prune_archives(
                     args.verbosity,
+                    args.dry_run,
                     repository,
+                    storage,
                     retention,
                     local_path=local_path,
                     remote_path=remote_path,
                 )
             if args.create:
-                logger.info('{}: Creating archive'.format(repository))
+                logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
                 create.create_archive(
                     args.verbosity,
+                    args.dry_run,
                     repository,
                     location,
                     storage,
@@ -124,6 +134,7 @@ def run_configuration(config_filename, args):  # pragma: no cover
                 check.check_archives(
                     args.verbosity,
                     repository,
+                    storage,
                     consistency,
                     local_path=local_path,
                     remote_path=remote_path,

borgmatic/config/schema.yaml

@@ -43,7 +43,8 @@ map:
                     - type: scalar
                 desc: |
                     Paths to local or remote repositories (required). Tildes are expanded. Multiple
-                    repositories are backed up to in sequence.
+                    repositories are backed up to in sequence. See ssh_command for SSH options like
+                    identity file or port.
                 example:
                     - user@backupserver:sourcehostname.borg
             patterns:
@@ -51,9 +52,9 @@ map:
                     - type: scalar
                 desc: |
                     Any paths matching these patterns are included/excluded from backups. Globs are
-                    expanded. Note that Borg considers this option experimental. See the output of
-                    "borg help patterns" for more details. Quoting any value if it contains leading
-                    punctuation, so it parses correctly.
+                    expanded. (Tildes are not.) Note that Borg considers this option experimental.
+                    See the output of "borg help patterns" for more details. Quote any value if it
+                    contains leading punctuation, so it parses correctly.
                 example:
                     - 'R /'
                     - '- /home/*/.cache'
@@ -72,11 +73,11 @@ map:
                 seq:
                     - type: scalar
                 desc: |
-                    Any paths matching these patterns are excluded from backups. Globs are expanded.
-                    See the output of "borg help patterns" for more details.
+                    Any paths matching these patterns are excluded from backups. Globs and tildes
+                    are expanded. See the output of "borg help patterns" for more details.
                 example:
                     - '*.pyc'
-                    - /home/*/.cache
+                    - ~/*/.cache
                     - /etc/ssl
             exclude_from:
                 seq:
@@ -103,6 +104,14 @@ map:
             https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for
             details.
         map:
+            encryption_passcommand:
+                type: scalar
+                desc: |
+                    The standard output of this command is used to unlock the encryption key. Only
+                    use on repositories that were initialized with passcommand/repokey encryption.
+                    Note that if both encryption_passcommand and encryption_passphrase are set,
+                    then encryption_passphrase takes precedence.
+                example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
                 type: scalar
                 desc: |
@@ -130,6 +139,10 @@ map:
                 type: scalar
                 desc: Umask to be used for borg create.
                 example: 0077
+            lock_wait:
+                type: int
+                desc: Maximum seconds to wait for acquiring a repository/cache lock.
+                example: 5
             archive_name_format:
                 type: scalar
                 desc: |
@@ -143,6 +156,7 @@ map:
         desc: |
             Retention policy for how many backups to keep in each category. See
             https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
+            At least one of the "keep" options is required for pruning to work.
         map:
             keep_within:
                 type: scalar

borgmatic/tests/unit/borg/test_check.py

@@ -103,6 +103,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -119,6 +120,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -136,6 +138,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
     module.check_archives(
         verbosity=VERBOSITY_SOME,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -153,6 +156,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
     module.check_archives(
         verbosity=VERBOSITY_LOTS,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -165,6 +169,7 @@ def test_check_archives_without_any_checks_bails():
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -186,6 +191,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
         local_path='borg1',
     )
@@ -208,6 +214,29 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
         remote_path='borg1',
     )
+
+
+def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    checks = ('repository',)
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--lock-wait', '5'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={'lock_wait': 5},
+        consistency_config=consistency_config,
+    )

borgmatic/tests/unit/borg/test_create.py

@@ -6,6 +6,17 @@ from borgmatic.borg import create as module
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
+def test_initialize_environment_with_passcommand_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize_environment({'encryption_passcommand': 'command'})
+        assert os.environ.get('BORG_PASSCOMMAND') == 'command'
+    finally:
+        os.environ = orig_environ
+
+
 def test_initialize_environment_with_passphrase_should_set_environment():
     orig_environ = os.environ
 
@@ -34,6 +45,7 @@ def test_initialize_environment_without_configuration_should_not_set_environment
     try:
         os.environ = {}
         module.initialize_environment({})
+        assert os.environ.get('BORG_PASSCOMMAND') == None
         assert os.environ.get('BORG_PASSPHRASE') == None
         assert os.environ.get('BORG_RSH') == None
     finally:
@@ -58,6 +70,21 @@ def test_expand_directory_with_glob_expands():
     assert paths == ['foo', 'food']
 
 
+def test_expand_directories_flattens_expanded_directories():
+    flexmock(module).should_receive('_expand_directory').with_args('~/foo').and_return(['/root/foo'])
+    flexmock(module).should_receive('_expand_directory').with_args('bar*').and_return(['bar', 'barf'])
+
+    paths = module._expand_directories(('~/foo', 'bar*'))
+
+    assert paths == ('/root/foo', 'bar', 'barf')
+
+
+def test_expand_directories_considers_none_as_no_directories():
+    paths = module._expand_directories(None)
+
+    assert paths == ()
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(
         name='filename',
@@ -84,7 +111,7 @@ def test_make_pattern_flags_includes_pattern_filename_when_given():
         pattern_filename='/tmp/patterns',
     )
 
-    assert pattern_flags == ('--pattern-from', '/tmp/patterns')
+    assert pattern_flags == ('--patterns-from', '/tmp/patterns')
 
 
 def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config():
@@ -92,7 +119,7 @@ def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config():
         location_config={'patterns_from': ['patterns', 'other']},
     )
 
-    assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', 'other')
+    assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', 'other')
 
 
 def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config():
@@ -101,7 +128,7 @@ def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patt
         pattern_filename='/tmp/patterns',
     )
 
-    assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', '/tmp/patterns')
+    assert pattern_flags == ('--patterns-from', 'patterns', '--patterns-from', '/tmp/patterns')
 
 
 def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty():
@@ -182,7 +209,7 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo
 
 
 def test_create_archive_calls_borg_with_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -190,6 +217,7 @@ def test_create_archive_calls_borg_with_parameters():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -202,7 +230,7 @@ def test_create_archive_calls_borg_with_parameters():
 
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -210,6 +238,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -222,7 +251,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes'))
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
@@ -230,6 +259,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -241,7 +271,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 
 def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -250,6 +280,7 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
 
     module.create_archive(
         verbosity=VERBOSITY_SOME,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -261,7 +292,7 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
 
 
 def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -269,6 +300,70 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
 
     module.create_archive(
         verbosity=VERBOSITY_LOTS,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--dry-run',))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=True,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_dry_run_and_verbosity_some_calls_borg_without_stats_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--info', '--dry-run'))
+
+    module.create_archive(
+        verbosity=VERBOSITY_SOME,
+        dry_run=True,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
+def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--dry-run'))
+
+    module.create_archive(
+        verbosity=VERBOSITY_LOTS,
+        dry_run=True,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -280,7 +375,7 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
 
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -288,6 +383,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -299,7 +395,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -307,6 +403,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -318,7 +415,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -326,6 +423,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -338,7 +436,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -346,6 +444,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -358,7 +457,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -366,6 +465,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -378,7 +478,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -386,6 +486,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -398,7 +499,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -406,6 +507,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -416,8 +518,28 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     )
 
 
+def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'lock_wait': 5},
+    )
+
+
 def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -426,6 +548,7 @@ def test_create_archive_with_source_directories_glob_expands():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo*'],
@@ -437,7 +560,7 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo*'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -446,6 +569,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo*'],
@@ -457,7 +581,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -465,6 +589,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo*'],
@@ -476,7 +601,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -484,6 +609,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],
@@ -497,7 +623,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([])
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -505,6 +631,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
 
     module.create_archive(
         verbosity=None,
+        dry_run=False,
         repository='repo',
         location_config={
             'source_directories': ['foo', 'bar'],

borgmatic/tests/unit/borg/test_extract.py

@@ -34,6 +34,7 @@ def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -48,6 +49,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_bail():
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -64,6 +66,7 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_
     module.extract_last_archive_dry_run(
         verbosity=VERBOSITY_SOME,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -80,6 +83,7 @@ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_
     module.extract_last_archive_dry_run(
         verbosity=VERBOSITY_LOTS,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -96,6 +100,7 @@ def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
         local_path='borg1',
     )
 
@@ -113,5 +118,23 @@ def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_paramete
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
         remote_path='borg1',
     )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        lock_wait=5,
+    )

borgmatic/tests/unit/borg/test_prune.py

@@ -64,7 +64,9 @@ def test_prune_archives_calls_borg_with_parameters():
 
     module.prune_archives(
         verbosity=None,
+        dry_run=False,
         repository='repo',
+        storage_config={},
         retention_config=retention_config,
     )
 
@@ -78,7 +80,9 @@ def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter():
 
     module.prune_archives(
         repository='repo',
+        storage_config={},
         verbosity=VERBOSITY_SOME,
+        dry_run=False,
         retention_config=retention_config,
     )
 
@@ -92,7 +96,25 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
 
     module.prune_archives(
         repository='repo',
+        storage_config={},
         verbosity=VERBOSITY_LOTS,
+        dry_run=False,
+        retention_config=retention_config,
+    )
+
+
+def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--dry-run',))
+
+    module.prune_archives(
+        repository='repo',
+        storage_config={},
+        verbosity=None,
+        dry_run=True,
         retention_config=retention_config,
     )
 
@@ -106,7 +128,9 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path():
 
     module.prune_archives(
         verbosity=None,
+        dry_run=False,
         repository='repo',
+        storage_config={},
         retention_config=retention_config,
         local_path='borg1',
     )
@@ -121,7 +145,26 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
 
     module.prune_archives(
         verbosity=None,
+        dry_run=False,
         repository='repo',
+        storage_config={},
         retention_config=retention_config,
         remote_path='borg1',
     )
+
+
+def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
+
+    module.prune_archives(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        storage_config=storage_config,
+        retention_config=retention_config,
+    )

scripts/push

@@ -2,5 +2,5 @@
 
 set -e
 
-git push -u origin
-git push -u github
+git push -u origin master
+git push -u github master

scripts/release

@@ -2,6 +2,11 @@
 
 set -e
 
+version=$(head --lines=1 NEWS)
+git tag $version
+git push origin $version
+git push github $version
+
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.13'
+VERSION = '1.1.15'
 
 
 setup(