diff --git a/NEWS b/NEWS index 7658fe71..c8d9110f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 2.0.13.dev0 + * #1179: Add a "file_list_format" option for setting the "list" action's output format and an + "archive_list_format" option for setting the "repo-list" action's format. * #1192: Fix for over-aggressive deduplication of source directories that contain the borgmatic runtime directory, potentially resulting in data loss (data not getting backed up) when snapshotting these source directories. diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 1f8d13b6..dd65608f 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -18,6 +18,7 @@ MAKE_FLAGS_EXCLUDES = ( 'archive', 'paths', 'find_paths', + 'format', *ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST, ) @@ -54,6 +55,7 @@ def make_list_command( + flags.make_flags('umask', config.get('umask')) + flags.make_flags('log-json', config.get('log_json')) + flags.make_flags('lock-wait', config.get('lock_wait')) + + flags.make_flags('format', list_arguments.format or config.get('file_list_format')) + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ()) + ( diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index c8d4e510..41238a1a 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -110,7 +110,7 @@ def get_latest_archive( return latest_archive -MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives') +MAKE_FLAGS_EXCLUDES = ('repository', 'format', 'prefix', 'match_archives') def make_repo_list_command( @@ -170,6 +170,9 @@ def make_repo_list_command( ) ) ) + + flags.make_flags( + 'format', repo_list_arguments.format or config.get('archive_list_format') + ) + flags.make_flags_from_arguments(repo_list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 40b540f9..4b2e94b2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -297,7 +297,14 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'} +OMITTED_FLAG_NAMES = { + 'match-archives', + 'progress', + 'statistics', + 'list-details', + 'file-list-format', + 'archive-list-format', +} def make_argument_description(schema, flag_name): @@ -1541,7 +1548,7 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 action='store_true', help='Output only archive names', ) - repo_list_group.add_argument('--format', help='Format for archive listing') + repo_list_group.add_argument('--format', help='Borg format for the archive listing') repo_list_group.add_argument( '--json', default=False, @@ -1644,7 +1651,7 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 action='store_true', help='Output only path names', ) - list_group.add_argument('--format', help='Format for file listing') + list_group.add_argument('--format', help='Borg format for the file listing') list_group.add_argument( '--json', default=False, diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0cbd4b0c..a21e15a3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -492,6 +492,27 @@ properties: If match_archives is not specified, borgmatic defaults to deriving the match_archives value from archive_name_format. example: "sh:{hostname}-*" + file_list_format: + type: string + description: | + Borg format for the files listing of the "list" action. Corresponds + to the "--format" flag. Defaults to "{mode} {user:6} {group:6} + {size:8} {mtime} {path}{extra}{NL}". With "--json", the form of the + format is ignored, but the keys used in it are added to the JSON + output. See + https://borgbackup.readthedocs.io/en/stable/usage/list.html for + details. + example: "{path}{extra}{NL}" + archive_list_format: + type: string + description: | + Borg format for the archives listing of the "repo-list" action. + Corresponds to the "--format" flag. Defaults to "{archive:<36} + {time} [{id}]{NL}". With "--json", the form of the format is + ignored, but the keys used in it are added to the JSON output. See + https://borgbackup.readthedocs.io/en/stable/usage/list.html for + details. + example: "{archive}{NL}" relocated_repo_access_is_ok: type: boolean description: | @@ -998,6 +1019,8 @@ properties: type: string description: | Python format string used for log messages written to the log file. + See https://docs.python.org/3/library/logging.html (and specifically + the LogRecord attributes with "{}-formatting") for details. example: "[{asctime}] {levelname}: {prefix}{message}" monitoring_verbosity: type: integer diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 90e55fb7..744de75f 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -19,7 +19,7 @@ def test_make_list_command_includes_log_info(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), ) @@ -36,7 +36,7 @@ def test_make_list_command_includes_json_but_not_info(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=True), + list_arguments=flexmock(archive=None, paths=None, format=None, json=True), global_arguments=flexmock(), ) @@ -53,7 +53,7 @@ def test_make_list_command_includes_log_debug(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), ) @@ -70,7 +70,7 @@ def test_make_list_command_includes_json_but_not_debug(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=True), + list_arguments=flexmock(archive=None, paths=None, format=None, json=True), global_arguments=flexmock(), ) @@ -86,7 +86,7 @@ def test_make_list_command_includes_json(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=True), + list_arguments=flexmock(archive=None, paths=None, format=None, json=True), global_arguments=flexmock(), ) @@ -96,7 +96,7 @@ def test_make_list_command_includes_json(): def test_make_list_command_includes_log_json(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--log-json',), - ) + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -104,7 +104,7 @@ def test_make_list_command_includes_log_json(): repository_path='repo', config={'log_json': True}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), ) @@ -114,7 +114,7 @@ def test_make_list_command_includes_log_json(): def test_make_list_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--lock-wait', '5'), - ) + ).and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -122,13 +122,31 @@ def test_make_list_command_includes_lock_wait(): repository_path='repo', config={'lock_wait': 5}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') +def test_make_list_command_includes_format(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( + () + ).and_return(('--format', 'stuff')) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_list_command( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + list_arguments=flexmock(archive=None, paths=None, format='stuff', json=False), + global_arguments=flexmock(), + ) + + assert command == ('borg', 'list', '--format', 'stuff', 'repo') + + def test_make_list_command_includes_extra_borg_options(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) @@ -138,7 +156,7 @@ def test_make_list_command_includes_extra_borg_options(): repository_path='repo', config={'extra_borg_options': {'list': '--extra "value with space"'}}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), ) @@ -156,7 +174,7 @@ def test_make_list_command_includes_archive(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive='archive', paths=None, json=False), + list_arguments=flexmock(archive='archive', paths=None, format=None, json=False), global_arguments=flexmock(), ) @@ -174,7 +192,7 @@ def test_make_list_command_includes_archive_and_path(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), + list_arguments=flexmock(archive='archive', paths=['var/lib'], format=None, json=False), global_arguments=flexmock(), ) @@ -190,7 +208,7 @@ def test_make_list_command_includes_local_path(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), local_path='borg2', ) @@ -214,7 +232,7 @@ def test_make_list_command_includes_remote_path(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), remote_path='borg2', ) @@ -233,7 +251,7 @@ def test_make_list_command_includes_umask(): repository_path='repo', config={'umask': '077'}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False), global_arguments=flexmock(), ) @@ -249,7 +267,7 @@ def test_make_list_command_includes_short(): repository_path='repo', config={}, local_borg_version='1.2.3', - list_arguments=flexmock(archive=None, paths=None, json=False, short=True), + list_arguments=flexmock(archive=None, paths=None, format=None, json=False, short=True), global_arguments=flexmock(), ) @@ -284,9 +302,9 @@ def test_make_list_command_includes_additional_flags(argument_name): list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, find_paths=None, - format=None, **{argument_name: 'value'}, ), global_arguments=flexmock(), @@ -344,6 +362,7 @@ def test_list_archive_calls_borg_with_flags(): list_arguments = argparse.Namespace( archive='archive', paths=None, + format=None, json=False, find_paths=None, prefix=None, @@ -410,6 +429,7 @@ def test_list_archive_calls_borg_with_local_path(): list_arguments = argparse.Namespace( archive='archive', paths=None, + format=None, json=False, find_paths=None, prefix=None, diff --git a/tests/unit/borg/test_repo_list.py b/tests/unit/borg/test_repo_list.py index 796e552b..faf4c152 100644 --- a/tests/unit/borg/test_repo_list.py +++ b/tests/unit/borg/test_repo_list.py @@ -600,6 +600,7 @@ def test_make_repo_list_command_includes_log_info(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -629,6 +630,7 @@ def test_make_repo_list_command_includes_json_but_not_info(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=True, prefix=None, match_archives=None, @@ -658,6 +660,7 @@ def test_make_repo_list_command_includes_log_debug(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -687,6 +690,7 @@ def test_make_repo_list_command_includes_json_but_not_debug(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=True, prefix=None, match_archives=None, @@ -715,6 +719,7 @@ def test_make_repo_list_command_includes_json(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=True, prefix=None, match_archives=None, @@ -729,7 +734,7 @@ def test_make_repo_list_command_includes_log_json(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--log-json',), - ).and_return(()) + ).and_return(()).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( None, None, @@ -745,6 +750,7 @@ def test_make_repo_list_command_includes_log_json(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -759,7 +765,7 @@ def test_make_repo_list_command_includes_lock_wait(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5'), - ).and_return(()) + ).and_return(()).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( None, None, @@ -775,6 +781,7 @@ def test_make_repo_list_command_includes_lock_wait(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -803,6 +810,7 @@ def test_make_repo_list_command_includes_list_extra_borg_options(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -831,6 +839,7 @@ def test_make_repo_list_command_with_feature_available_includes_repo_list_extra_ repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -859,6 +868,7 @@ def test_make_repo_list_command_includes_local_path(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -890,6 +900,7 @@ def test_make_repo_list_command_includes_remote_path(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -921,6 +932,7 @@ def test_make_repo_list_command_includes_umask(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -935,7 +947,7 @@ def test_make_repo_list_command_transforms_prefix_into_match_archives(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--match-archives', 'sh:foo*'), - ) + ).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( None, None, @@ -948,7 +960,9 @@ def test_make_repo_list_command_transforms_prefix_into_match_archives(): repository_path='repo', config={}, local_borg_version='1.2.3', - repo_list_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + repo_list_arguments=flexmock( + archive=None, paths=None, format=None, json=False, prefix='foo' + ), global_arguments=flexmock(), ) @@ -959,7 +973,7 @@ def test_make_repo_list_command_prefers_prefix_over_archive_name_format(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( ('--match-archives', 'sh:foo*'), - ) + ).and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').never() flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -968,7 +982,9 @@ def test_make_repo_list_command_prefers_prefix_over_archive_name_format(): repository_path='repo', config={'archive_name_format': 'bar-{now}'}, local_borg_version='1.2.3', - repo_list_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + repo_list_arguments=flexmock( + archive=None, paths=None, format=None, json=False, prefix='foo' + ), global_arguments=flexmock(), ) @@ -993,6 +1009,7 @@ def test_make_repo_list_command_transforms_archive_name_format_into_match_archiv repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -1003,6 +1020,38 @@ def test_make_repo_list_command_transforms_archive_name_format_into_match_archiv assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') +def test_make_repo_list_command_includes_format_from_command_line(): + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return( + ('--format', 'stuff') + ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, + None, + '1.2.3', + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_repo_list_command( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + repo_list_arguments=flexmock( + archive=None, + paths=None, + format='stuff', + json=False, + prefix=None, + match_archives=None, + short=False, + ), + global_arguments=flexmock(), + ) + + assert command == ('borg', 'list', '--format', 'stuff', 'repo') + + def test_make_repo_list_command_includes_short(): flexmock(module.feature).should_receive('available').and_return(False) flexmock(module.flags).should_receive('make_flags').and_return(()) @@ -1021,6 +1070,7 @@ def test_make_repo_list_command_includes_short(): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, @@ -1064,11 +1114,11 @@ def test_make_repo_list_command_includes_additional_flags(argument_name): repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None, find_paths=None, - format=None, **{argument_name: 'value'}, ), global_arguments=flexmock(), @@ -1100,11 +1150,11 @@ def test_make_repo_list_command_with_match_archives_calls_borg_with_match_archiv repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives='foo-*', find_paths=None, - format=None, ), global_arguments=flexmock(), ) @@ -1173,6 +1223,7 @@ def test_make_repo_list_command_with_date_based_matching_calls_borg_with_date_ba repo_list_arguments=flexmock( archive=None, paths=None, + format=None, json=False, prefix=None, match_archives=None,