From 0e1659bd73f06a4f5ec609458253093a29258495 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Apr 2026 10:13:21 -0700 Subject: [PATCH] Expand the "patterns_from" and "exclude_from" options to support paths containing tildes and globs (#1301). --- NEWS | 2 ++ borgmatic/actions/check.py | 2 +- borgmatic/actions/create.py | 2 +- borgmatic/actions/diff.py | 5 +++-- borgmatic/actions/pattern.py | 13 ++++++----- borgmatic/actions/recreate.py | 6 +++-- borgmatic/actions/restore.py | 2 +- tests/unit/actions/test_pattern.py | 36 ++++++++++++++++++++++++------ 8 files changed, 49 insertions(+), 19 deletions(-) diff --git a/NEWS b/NEWS index 7ad2f218..4a80fbeb 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 2.1.6.dev0 * #1300: Fix the "source_directories_must_exist" option to support source directories relative to a "working_directory". + * #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes + and globs. * Add the NEWS changelog file to release tarball (#1298). * Enable reply by email on projects.torsion.org, so replies to notification emails get posted as comments on tickets. diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 28f5e8f6..7967aaf3 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -383,7 +383,7 @@ def collect_spot_check_source_paths( # Omit "progress" because it interferes with "list_details". config=dict(config, progress=False, list_details=True), patterns=borgmatic.actions.pattern.process_patterns( - borgmatic.actions.pattern.collect_patterns(config) + borgmatic.actions.pattern.collect_patterns(config, working_directory) + tuple( borgmatic.borg.pattern.Pattern( config_path, diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 86310085..e307ffd7 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -45,7 +45,7 @@ def run_create( with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory: patterns = pattern.process_patterns( - pattern.collect_patterns(config), + pattern.collect_patterns(config, working_directory), config, working_directory, borgmatic_runtime_directory, diff --git a/borgmatic/actions/diff.py b/borgmatic/actions/diff.py index 32b0cf5b..42c4c327 100644 --- a/borgmatic/actions/diff.py +++ b/borgmatic/actions/diff.py @@ -21,10 +21,11 @@ def run_diff( # Only process patterns if only_patterns flag is set if diff_arguments.only_patterns: + working_directory = borgmatic.config.paths.get_working_directory(config) processed_patterns = borgmatic.actions.pattern.process_patterns( - (*borgmatic.actions.pattern.collect_patterns(config),), + (*borgmatic.actions.pattern.collect_patterns(config, working_directory),), config, - borgmatic.config.paths.get_working_directory(config), + working_directory, ) else: processed_patterns = None diff --git a/borgmatic/actions/pattern.py b/borgmatic/actions/pattern.py index 6c2ced3f..16d35182 100644 --- a/borgmatic/actions/pattern.py +++ b/borgmatic/actions/pattern.py @@ -34,10 +34,11 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty ) -def collect_patterns(config): +def collect_patterns(config, working_directory): ''' - Given a configuration dict, produce a single sequence of patterns comprised of the configured - source directories, patterns, excludes, pattern files, and exclude files. + Given a configuration dict and the working directory, produce a single sequence of patterns + comprised of the configured source directories, patterns, excludes, pattern files, and exclude + files. The idea is that Borg has all these different ways of specifying includes, excludes, source directories, etc., but we'd like to collapse them all down to one common format (patterns) for @@ -68,7 +69,8 @@ def collect_patterns(config): + tuple( parse_pattern(pattern_line.strip()) for filename in config.get('patterns_from', ()) - for pattern_line in open(filename, encoding='utf-8') + for expanded_path in expand_directory(filename, working_directory) + for pattern_line in open(expanded_path, encoding='utf-8') if not pattern_line.lstrip().startswith('#') if pattern_line.strip() ) @@ -78,7 +80,8 @@ def collect_patterns(config): borgmatic.borg.pattern.Pattern_style.FNMATCH, ) for filename in config.get('exclude_from', ()) - for exclude_line in open(filename, encoding='utf-8') + for expanded_path in expand_directory(filename, working_directory) + for exclude_line in open(expanded_path, encoding='utf-8') if not exclude_line.lstrip().startswith('#') if exclude_line.strip() ) diff --git a/borgmatic/actions/recreate.py b/borgmatic/actions/recreate.py index 06549843..c6360a41 100644 --- a/borgmatic/actions/recreate.py +++ b/borgmatic/actions/recreate.py @@ -30,10 +30,12 @@ def run_recreate( else: logger.answer(f'Recreating repository{dry_run_label}') + working_directory = borgmatic.config.paths.get_working_directory(config) + # Collect and process patterns. processed_patterns = borgmatic.actions.pattern.process_patterns( ( - *borgmatic.actions.pattern.collect_patterns(config), + *borgmatic.actions.pattern.collect_patterns(config, working_directory), # Also add borgmatic-specific paths, so they don't get excluded from the recreated # archive. Note that this doesn't currently work for archives created with Borg 1.2 or # below. @@ -42,7 +44,7 @@ def run_recreate( ), ), config, - borgmatic.config.paths.get_working_directory(config), + working_directory, ) archive = borgmatic.borg.repo_list.resolve_archive_name( diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 6442880a..3f38b843 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -539,7 +539,7 @@ def run_restore( with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory: patterns = borgmatic.actions.pattern.process_patterns( - borgmatic.actions.pattern.collect_patterns(config), + borgmatic.actions.pattern.collect_patterns(config, working_directory), config, working_directory, ) diff --git a/tests/unit/actions/test_pattern.py b/tests/unit/actions/test_pattern.py index 03e02c2e..6f7879c0 100644 --- a/tests/unit/actions/test_pattern.py +++ b/tests/unit/actions/test_pattern.py @@ -35,7 +35,7 @@ def test_parse_pattern_with_invalid_pattern_line_errors(): def test_collect_patterns_converts_source_directories(): - assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == ( + assert module.collect_patterns({'source_directories': ['/foo', '/bar']}, '/working') == ( Pattern('/foo', source=Pattern_source.CONFIG), Pattern('/bar', source=Pattern_source.CONFIG), ) @@ -48,14 +48,18 @@ def test_collect_patterns_parses_config_patterns(): flexmock(module).should_receive('parse_pattern').with_args(' ').never() flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar')) - assert module.collect_patterns({'patterns': ['R /foo', '# comment', '', ' ', 'R /bar']}) == ( + assert module.collect_patterns( + {'patterns': ['R /foo', '# comment', '', ' ', 'R /bar']}, '/working' + ) == ( Pattern('/foo'), Pattern('/bar'), ) def test_collect_patterns_converts_exclude_patterns(): - assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == ( + assert module.collect_patterns( + {'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}, '/working' + ) == ( Pattern( '/foo', Pattern_type.NO_RECURSE, @@ -78,6 +82,12 @@ def test_collect_patterns_converts_exclude_patterns(): def test_collect_patterns_reads_config_patterns_from_file(): + flexmock(module).should_receive('expand_directory').with_args( + 'file1.txt', '/working' + ).and_return(['file1.txt']) + flexmock(module).should_receive('expand_directory').with_args( + 'file2.txt', '/working' + ).and_return(['file2.txt']) builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_return( io.StringIO('R /foo') @@ -92,7 +102,7 @@ def test_collect_patterns_reads_config_patterns_from_file(): flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar')) flexmock(module).should_receive('parse_pattern').with_args('R /baz').and_return(Pattern('/baz')) - assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}) == ( + assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}, '/working') == ( Pattern('/foo'), Pattern('/bar'), Pattern('/baz'), @@ -101,16 +111,25 @@ def test_collect_patterns_reads_config_patterns_from_file(): def test_collect_patterns_errors_on_missing_config_patterns_from_file(): builtins = flexmock(sys.modules['builtins']) + flexmock(module).should_receive('expand_directory').with_args( + 'file1.txt', '/working' + ).and_return(['file1.txt']) builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_raise( FileNotFoundError ) flexmock(module).should_receive('parse_pattern').never() with pytest.raises(ValueError): - module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}) + module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}, '/working') def test_collect_patterns_reads_config_exclude_from_file(): + flexmock(module).should_receive('expand_directory').with_args( + 'file1.txt', '/working' + ).and_return(['file1.txt']) + flexmock(module).should_receive('expand_directory').with_args( + 'file2.txt', '/working' + ).and_return(['file2.txt']) builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_return( io.StringIO('/foo') @@ -134,7 +153,7 @@ def test_collect_patterns_reads_config_exclude_from_file(): default_style=Pattern_style.FNMATCH, ).and_return(Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH)) - assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) == ( + assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}, '/working') == ( Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), @@ -142,12 +161,15 @@ def test_collect_patterns_reads_config_exclude_from_file(): def test_collect_patterns_errors_on_missing_config_exclude_from_file(): + flexmock(module).should_receive('expand_directory').with_args( + 'file1.txt', '/working' + ).and_return(['file1.txt']) builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_raise(OSError) flexmock(module).should_receive('parse_pattern').never() with pytest.raises(ValueError): - module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) + module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}, '/working') def test_expand_directory_with_basic_path_passes_it_through():