Filter listed paths via "borgmatic list --path" flag (#269).

NEWS

@@ -1,3 +1,10 @@
+1.4.18
+ * Fix "--repository" flag to accept relative paths.
+ * Fix "borgmatic umount" so it only runs Borg once instead of once per repository / configuration
+   file.
+ * #253: Mount whole repositories via "borgmatic mount" without any "--archive" flag.
+ * #269: Filter listed paths via "borgmatic list --path" flag.
+
 1.4.17
  * #235: Pass extra options directly to particular Borg commands, handy for Borg options that
    borgmatic does not yet support natively. Use "extra_borg_options" in the storage configuration

borgmatic/borg/list.py

@@ -36,13 +36,14 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
         + make_flags('remote-path', remote_path)
         + make_flags('lock-wait', lock_wait)
         + make_flags_from_arguments(
-            list_arguments, excludes=('repository', 'archive', 'successful')
+            list_arguments, excludes=('repository', 'archive', 'paths', 'successful')
         )
         + (
             '::'.join((repository, list_arguments.archive))
             if list_arguments.archive
             else repository,
         )
+        + (tuple(list_arguments.paths) if list_arguments.paths else ())
     )
 
     return execute_command(

borgmatic/borg/mount.py

@@ -17,9 +17,9 @@ def mount_archive(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, an archive name, a filesystem mount point, zero or more
-    paths to mount from the archive, extra Borg mount options, a storage configuration dict, and
-    optional local and remote Borg paths, mount the archive onto the mount point.
+    Given a local or remote repository path, an optional archive name, a filesystem mount point,
+    zero or more paths to mount from the archive, extra Borg mount options, a storage configuration
+    dict, and optional local and remote Borg paths, mount the archive onto the mount point.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
@@ -33,7 +33,7 @@ def mount_archive(
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--foreground',) if foreground else ())
         + (('-o', options) if options else ())
-        + ('::'.join((repository, archive)),)
+        + (('::'.join((repository, archive)),) if archive else (repository,))
         + (mount_point,)
         + (tuple(paths) if paths else ())
     )

borgmatic/commands/arguments.py

@@ -333,7 +333,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to use, defaults to the configured repository if there is only one',
     )
-    mount_group.add_argument('--archive', help='Name of archive to mount', required=True)
+    mount_group.add_argument('--archive', help='Name of archive to mount')
     mount_group.add_argument(
         '--mount-point',
         metavar='PATH',
@@ -419,6 +419,13 @@ def parse_arguments(*unparsed_arguments):
         help='Path of repository to list, defaults to the configured repository if there is only one',
     )
     list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument(
+        '--path',
+        metavar='PATH',
+        nargs='+',
+        dest='paths',
+        help='Paths to list from archive, defaults to the entire archive',
+    )
     list_group.add_argument(
         '--short', default=False, action='store_true', help='Output only archive or path names'
     )

borgmatic/commands/borgmatic.py

@@ -234,7 +234,9 @@ def run_actions(
             only_checks=arguments['check'].only,
         )
     if 'extract' in arguments:
-        if arguments['extract'].repository is None or repository == arguments['extract'].repository:
+        if arguments['extract'].repository is None or validate.repositories_match(
+            repository, arguments['extract'].repository
+        ):
             logger.info(
                 '{}: Extracting archive {}'.format(repository, arguments['extract'].archive)
             )
@@ -251,8 +253,16 @@ def run_actions(
                 progress=arguments['extract'].progress,
             )
     if 'mount' in arguments:
-        if arguments['mount'].repository is None or repository == arguments['mount'].repository:
-            logger.info('{}: Mounting archive {}'.format(repository, arguments['mount'].archive))
+        if arguments['mount'].repository is None or validate.repositories_match(
+            repository, arguments['mount'].repository
+        ):
+            if arguments['mount'].archive:
+                logger.info(
+                    '{}: Mounting archive {}'.format(repository, arguments['mount'].archive)
+                )
+            else:
+                logger.info('{}: Mounting repository'.format(repository))
+
             borg_mount.mount_archive(
                 repository,
                 arguments['mount'].archive,
@@ -264,15 +274,10 @@ def run_actions(
                 local_path=local_path,
                 remote_path=remote_path,
             )
-    if 'umount' in arguments:
-        logger.info(
-            '{}: Unmounting mount point {}'.format(repository, arguments['umount'].mount_point)
-        )
-        borg_umount.unmount_archive(
-            mount_point=arguments['umount'].mount_point, local_path=local_path
-        )
     if 'restore' in arguments:
-        if arguments['restore'].repository is None or repository == arguments['restore'].repository:
+        if arguments['restore'].repository is None or validate.repositories_match(
+            repository, arguments['restore'].repository
+        ):
             logger.info(
                 '{}: Restoring databases from archive {}'.format(
                     repository, arguments['restore'].archive
@@ -330,7 +335,9 @@ def run_actions(
                 global_arguments.dry_run,
             )
     if 'list' in arguments:
-        if arguments['list'].repository is None or repository == arguments['list'].repository:
+        if arguments['list'].repository is None or validate.repositories_match(
+            repository, arguments['list'].repository
+        ):
             logger.info('{}: Listing archives'.format(repository))
             json_output = borg_list.list_archives(
                 repository,
@@ -342,7 +349,9 @@ def run_actions(
             if json_output:
                 yield json.loads(json_output)
     if 'info' in arguments:
-        if arguments['info'].repository is None or repository == arguments['info'].repository:
+        if arguments['info'].repository is None or validate.repositories_match(
+            repository, arguments['info'].repository
+        ):
             logger.info('{}: Displaying summary info for archives'.format(repository))
             json_output = borg_info.display_archives_info(
                 repository,
@@ -431,6 +440,14 @@ def make_error_log_records(message, error=None):
         pass
 
 
+def get_local_path(configs):
+    '''
+    Arbitrarily return the local path from the first configuration dict. Default to "borg" if not
+    set.
+    '''
+    return next(iter(configs.values())).get('location', {}).get('local_path', 'borg')
+
+
 def collect_configuration_run_summary_logs(configs, arguments):
     '''
     Given a dict of configuration filename to corresponding parsed configuration, and parsed
@@ -501,6 +518,15 @@ def collect_configuration_run_summary_logs(configs, arguments):
             if results:
                 json_results.extend(results)
 
+    if 'umount' in arguments:
+        logger.info('Unmounting mount point {}'.format(arguments['umount'].mount_point))
+        try:
+            borg_umount.unmount_archive(
+                mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs)
+            )
+        except (CalledProcessError, OSError) as error:
+            yield from make_error_log_records('Error unmounting mount point', error)
+
     if json_results:
         sys.stdout.write(json.dumps(json_results))
 

borgmatic/config/validate.py

@@ -1,4 +1,5 @@
 import logging
+import os
 
 import pkg_resources
 import pykwalify.core
@@ -112,6 +113,24 @@ def parse_configuration(config_filename, schema_filename):
     return parsed_result
 
 
+def normalize_repository_path(repository):
+    '''
+    Given a repository path, return the absolute path of it (for local repositories).
+    '''
+    # A colon in the repository indicates it's a remote repository. Bail.
+    if ':' in repository:
+        return repository
+
+    return os.path.abspath(repository)
+
+
+def repositories_match(first, second):
+    '''
+    Given two repository paths (relative and/or absolute), return whether they match.
+    '''
+    return normalize_repository_path(first) == normalize_repository_path(second)
+
+
 def guard_configuration_contains_repository(repository, configurations):
     '''
     Given a repository path and a dict mapping from config filename to corresponding parsed config
@@ -133,9 +152,7 @@ def guard_configuration_contains_repository(repository, configurations):
 
         if count > 1:
             raise ValueError(
-                'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
-                    repository
-                )
+                'Can\'t determine which repository to use. Use --repository option to disambiguate'
             )
 
         return
@@ -145,7 +162,7 @@ def guard_configuration_contains_repository(repository, configurations):
             config_repository
             for config in configurations.values()
             for config_repository in config['location']['repositories']
-            if repository == config_repository
+            if repositories_match(repository, config_repository)
         )
     )
 

docs/how-to/extract-a-backup.md

@@ -100,6 +100,12 @@ borgmatic mount --archive host-2019-... --mount-point /mnt
 This mounts the entire archive on the given mount point `/mnt`, so that you
 can look in there for your files.
 
+Omit the `--archive` flag to mount all archives (lazy-loaded):
+
+```bash
+borgmatic mount --mount-point /mnt
+```
+
 If you'd like to restrict the mounted filesystem to only particular paths from
 your archive, use the `--path` flag, similar to the `extract` action above.
 For instance:

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.4.17'
+VERSION = '1.4.18'
 
 
 setup(

tests/integration/commands/test_arguments.py

@@ -352,13 +352,6 @@ def test_parse_arguments_requires_archive_with_extract():
         module.parse_arguments('--config', 'myconfig', 'extract')
 
 
-def test_parse_arguments_requires_archive_with_mount():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(SystemExit):
-        module.parse_arguments('--config', 'myconfig', 'mount', '--mount-point', '/mnt')
-
-
 def test_parse_arguments_requires_archive_with_restore():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/unit/borg/test_list.py

@@ -16,7 +16,7 @@ def test_list_archives_calls_borg_with_parameters():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
     )
 
 
@@ -31,7 +31,7 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
     )
 
 
@@ -44,7 +44,7 @@ def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=True, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=True, successful=False),
     )
 
 
@@ -59,7 +59,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
     )
 
 
@@ -72,7 +72,7 @@ def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=True, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=True, successful=False),
     )
 
 
@@ -87,7 +87,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     module.list_archives(
         repository='repo',
         storage_config=storage_config,
-        list_arguments=flexmock(archive=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
     )
 
 
@@ -100,7 +100,22 @@ def test_list_archives_with_archive_calls_borg_with_archive_parameter():
     module.list_archives(
         repository='repo',
         storage_config=storage_config,
-        list_arguments=flexmock(archive='archive', json=False, successful=False),
+        list_arguments=flexmock(archive='archive', paths=None, json=False, successful=False),
+    )
+
+
+def test_list_archives_with_path_calls_borg_with_path_parameter():
+    storage_config = {}
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo::archive', 'var/lib'),
+        output_log_level=logging.WARNING,
+        error_on_warnings=False,
+    )
+
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False, successful=False),
     )
 
 
@@ -112,7 +127,7 @@ def test_list_archives_with_local_path_calls_borg_via_local_path():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
         local_path='borg1',
     )
 
@@ -127,7 +142,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
         remote_path='borg1',
     )
 
@@ -142,7 +157,7 @@ def test_list_archives_with_short_calls_borg_with_short_parameter():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=False, successful=False, short=True),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False, short=True),
     )
 
 
@@ -171,7 +186,7 @@ def test_list_archives_passes_through_arguments_to_borg(argument_name):
         repository='repo',
         storage_config={},
         list_arguments=flexmock(
-            archive=None, json=False, successful=False, **{argument_name: 'value'}
+            archive=None, paths=None, json=False, successful=False, **{argument_name: 'value'}
         ),
     )
 
@@ -186,7 +201,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
     module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=False, successful=True),
+        list_arguments=flexmock(archive=None, paths=None, json=False, successful=True),
     )
 
 
@@ -198,7 +213,7 @@ def test_list_archives_with_json_calls_borg_with_json_parameter():
     json_output = module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, json=True, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=True, successful=False),
     )
 
     assert json_output == '[]'

tests/unit/commands/test_borgmatic.py

@@ -169,6 +169,18 @@ def test_make_error_log_records_generates_nothing_for_other_error():
     assert logs == ()
 
 
+def test_get_local_path_uses_configuration_value():
+    assert module.get_local_path({'test.yaml': {'location': {'local_path': 'borg1'}}}) == 'borg1'
+
+
+def test_get_local_path_without_location_defaults_to_borg():
+    assert module.get_local_path({'test.yaml': {}}) == 'borg'
+
+
+def test_get_local_path_without_local_path_defaults_to_borg():
+    assert module.get_local_path({'test.yaml': {'location': {}}}) == 'borg'
+
+
 def test_collect_configuration_run_summary_logs_info_for_success():
     flexmock(module.command).should_receive('execute_hook').never()
     flexmock(module).should_receive('run_configuration').and_return([])
@@ -324,6 +336,22 @@ def test_collect_configuration_run_summary_logs_run_configuration_error():
     assert {log.levelno for log in logs} == {logging.CRITICAL}
 
 
+def test_collect_configuration_run_summary_logs_run_umount_error():
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository')
+    flexmock(module).should_receive('run_configuration').and_return([])
+    flexmock(module.borg_umount).should_receive('unmount_archive').and_raise(OSError)
+    flexmock(module).should_receive('make_error_log_records').and_return(
+        [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))]
+    )
+    arguments = {'umount': flexmock(mount_point='/mnt')}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert {log.levelno for log in logs} == {logging.INFO, logging.CRITICAL}
+
+
 def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
         ['baz']

tests/unit/config/test_validate.py

@@ -1,4 +1,5 @@
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import validate as module
 
@@ -95,7 +96,38 @@ def test_remove_examples_strips_examples_from_sequence_of_maps():
     assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]}
 
 
+def test_normalize_repository_path_passes_through_remote_repository():
+    repository = 'example.org:test.borg'
+
+    module.normalize_repository_path(repository) == repository
+
+
+def test_normalize_repository_path_passes_through_absolute_repository():
+    repository = '/foo/bar/test.borg'
+    flexmock(module.os.path).should_receive('abspath').and_return(repository)
+
+    module.normalize_repository_path(repository) == repository
+
+
+def test_normalize_repository_path_resolves_relative_repository():
+    repository = 'test.borg'
+    absolute = '/foo/bar/test.borg'
+    flexmock(module.os.path).should_receive('abspath').and_return(absolute)
+
+    module.normalize_repository_path(repository) == absolute
+
+
+def test_repositories_match_does_not_raise():
+    flexmock(module).should_receive('normalize_repository_path')
+
+    module.repositories_match('foo', 'bar')
+
+
 def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
+    flexmock(module).should_receive('repositories_match').replace_with(
+        lambda first, second: first == second
+    )
+
     module.guard_configuration_contains_repository(
         repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}}
     )
@@ -116,6 +148,10 @@ def test_guard_configuration_contains_repository_errors_when_repository_assumed_
 
 
 def test_guard_configuration_contains_repository_errors_when_repository_missing_from_config():
+    flexmock(module).should_receive('repositories_match').replace_with(
+        lambda first, second: first == second
+    )
+
     with pytest.raises(ValueError):
         module.guard_configuration_contains_repository(
             repository='nope',
@@ -124,6 +160,10 @@ def test_guard_configuration_contains_repository_errors_when_repository_missing_
 
 
 def test_guard_configuration_contains_repository_errors_when_repository_matches_config_twice():
+    flexmock(module).should_receive('repositories_match').replace_with(
+        lambda first, second: first == second
+    )
+
     with pytest.raises(ValueError):
         module.guard_configuration_contains_repository(
             repository='repo',