Expand the "patterns_from" and "exclude_from" options to support paths containing tildes and globs (#1301).

NEWS

@@ -1,6 +1,8 @@
 2.1.6.dev0
  * #1300: Fix the "source_directories_must_exist" option to support source directories relative to a
    "working_directory".
+ * #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes
+   and globs.
  * Add the NEWS changelog file to release tarball (#1298).
  * Enable reply by email on projects.torsion.org, so replies to notification emails get posted as
    comments on tickets.

borgmatic/actions/check.py

@@ -383,7 +383,7 @@ def collect_spot_check_source_paths(
         # Omit "progress" because it interferes with "list_details".
         config=dict(config, progress=False, list_details=True),
         patterns=borgmatic.actions.pattern.process_patterns(
-            borgmatic.actions.pattern.collect_patterns(config)
+            borgmatic.actions.pattern.collect_patterns(config, working_directory)
             + tuple(
                 borgmatic.borg.pattern.Pattern(
                     config_path,

borgmatic/actions/create.py

@@ -45,7 +45,7 @@ def run_create(
 
     with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
         patterns = pattern.process_patterns(
-            pattern.collect_patterns(config),
+            pattern.collect_patterns(config, working_directory),
             config,
             working_directory,
             borgmatic_runtime_directory,

borgmatic/actions/diff.py

@@ -21,10 +21,11 @@ def run_diff(
 
     # Only process patterns if only_patterns flag is set
     if diff_arguments.only_patterns:
+        working_directory = borgmatic.config.paths.get_working_directory(config)
         processed_patterns = borgmatic.actions.pattern.process_patterns(
-            (*borgmatic.actions.pattern.collect_patterns(config),),
+            (*borgmatic.actions.pattern.collect_patterns(config, working_directory),),
             config,
-            borgmatic.config.paths.get_working_directory(config),
+            working_directory,
         )
     else:
         processed_patterns = None

borgmatic/actions/pattern.py

@@ -34,10 +34,11 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty
     )
 
 
-def collect_patterns(config):
+def collect_patterns(config, working_directory):
     '''
-    Given a configuration dict, produce a single sequence of patterns comprised of the configured
-    source directories, patterns, excludes, pattern files, and exclude files.
+    Given a configuration dict and the working directory, produce a single sequence of patterns
+    comprised of the configured source directories, patterns, excludes, pattern files, and exclude
+    files.
 
     The idea is that Borg has all these different ways of specifying includes, excludes, source
     directories, etc., but we'd like to collapse them all down to one common format (patterns) for
@@ -68,7 +69,8 @@ def collect_patterns(config):
             + tuple(
                 parse_pattern(pattern_line.strip())
                 for filename in config.get('patterns_from', ())
-                for pattern_line in open(filename, encoding='utf-8')
+                for expanded_path in expand_directory(filename, working_directory)
+                for pattern_line in open(expanded_path, encoding='utf-8')
                 if not pattern_line.lstrip().startswith('#')
                 if pattern_line.strip()
             )
@@ -78,7 +80,8 @@ def collect_patterns(config):
                     borgmatic.borg.pattern.Pattern_style.FNMATCH,
                 )
                 for filename in config.get('exclude_from', ())
-                for exclude_line in open(filename, encoding='utf-8')
+                for expanded_path in expand_directory(filename, working_directory)
+                for exclude_line in open(expanded_path, encoding='utf-8')
                 if not exclude_line.lstrip().startswith('#')
                 if exclude_line.strip()
             )

borgmatic/actions/recreate.py

@@ -30,10 +30,12 @@ def run_recreate(
     else:
         logger.answer(f'Recreating repository{dry_run_label}')
 
+    working_directory = borgmatic.config.paths.get_working_directory(config)
+
     # Collect and process patterns.
     processed_patterns = borgmatic.actions.pattern.process_patterns(
         (
-            *borgmatic.actions.pattern.collect_patterns(config),
+            *borgmatic.actions.pattern.collect_patterns(config, working_directory),
             # Also add borgmatic-specific paths, so they don't get excluded from the recreated
             # archive. Note that this doesn't currently work for archives created with Borg 1.2 or
             # below.
@@ -42,7 +44,7 @@ def run_recreate(
             ),
         ),
         config,
-        borgmatic.config.paths.get_working_directory(config),
+        working_directory,
     )
 
     archive = borgmatic.borg.repo_list.resolve_archive_name(

borgmatic/actions/restore.py

@@ -539,7 +539,7 @@ def run_restore(
 
     with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
         patterns = borgmatic.actions.pattern.process_patterns(
-            borgmatic.actions.pattern.collect_patterns(config),
+            borgmatic.actions.pattern.collect_patterns(config, working_directory),
             config,
             working_directory,
         )

tests/unit/actions/test_pattern.py

@@ -35,7 +35,7 @@ def test_parse_pattern_with_invalid_pattern_line_errors():
 
 
 def test_collect_patterns_converts_source_directories():
-    assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == (
+    assert module.collect_patterns({'source_directories': ['/foo', '/bar']}, '/working') == (
         Pattern('/foo', source=Pattern_source.CONFIG),
         Pattern('/bar', source=Pattern_source.CONFIG),
     )
@@ -48,14 +48,18 @@ def test_collect_patterns_parses_config_patterns():
     flexmock(module).should_receive('parse_pattern').with_args('   ').never()
     flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
 
-    assert module.collect_patterns({'patterns': ['R /foo', '# comment', '', '   ', 'R /bar']}) == (
+    assert module.collect_patterns(
+        {'patterns': ['R /foo', '# comment', '', '   ', 'R /bar']}, '/working'
+    ) == (
         Pattern('/foo'),
         Pattern('/bar'),
     )
 
 
 def test_collect_patterns_converts_exclude_patterns():
-    assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
+    assert module.collect_patterns(
+        {'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}, '/working'
+    ) == (
         Pattern(
             '/foo',
             Pattern_type.NO_RECURSE,
@@ -78,6 +82,12 @@ def test_collect_patterns_converts_exclude_patterns():
 
 
 def test_collect_patterns_reads_config_patterns_from_file():
+    flexmock(module).should_receive('expand_directory').with_args(
+        'file1.txt', '/working'
+    ).and_return(['file1.txt'])
+    flexmock(module).should_receive('expand_directory').with_args(
+        'file2.txt', '/working'
+    ).and_return(['file2.txt'])
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_return(
         io.StringIO('R /foo')
@@ -92,7 +102,7 @@ def test_collect_patterns_reads_config_patterns_from_file():
     flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
     flexmock(module).should_receive('parse_pattern').with_args('R /baz').and_return(Pattern('/baz'))
 
-    assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}) == (
+    assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}, '/working') == (
         Pattern('/foo'),
         Pattern('/bar'),
         Pattern('/baz'),
@@ -101,16 +111,25 @@ def test_collect_patterns_reads_config_patterns_from_file():
 
 def test_collect_patterns_errors_on_missing_config_patterns_from_file():
     builtins = flexmock(sys.modules['builtins'])
+    flexmock(module).should_receive('expand_directory').with_args(
+        'file1.txt', '/working'
+    ).and_return(['file1.txt'])
     builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_raise(
         FileNotFoundError
     )
     flexmock(module).should_receive('parse_pattern').never()
 
     with pytest.raises(ValueError):
-        module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']})
+        module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}, '/working')
 
 
 def test_collect_patterns_reads_config_exclude_from_file():
+    flexmock(module).should_receive('expand_directory').with_args(
+        'file1.txt', '/working'
+    ).and_return(['file1.txt'])
+    flexmock(module).should_receive('expand_directory').with_args(
+        'file2.txt', '/working'
+    ).and_return(['file2.txt'])
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_return(
         io.StringIO('/foo')
@@ -134,7 +153,7 @@ def test_collect_patterns_reads_config_exclude_from_file():
         default_style=Pattern_style.FNMATCH,
     ).and_return(Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
 
-    assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) == (
+    assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}, '/working') == (
         Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
         Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
         Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
@@ -142,12 +161,15 @@ def test_collect_patterns_reads_config_exclude_from_file():
 
 
 def test_collect_patterns_errors_on_missing_config_exclude_from_file():
+    flexmock(module).should_receive('expand_directory').with_args(
+        'file1.txt', '/working'
+    ).and_return(['file1.txt'])
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('file1.txt', encoding='utf-8').and_raise(OSError)
     flexmock(module).should_receive('parse_pattern').never()
 
     with pytest.raises(ValueError):
-        module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']})
+        module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}, '/working')
 
 
 def test_expand_directory_with_basic_path_passes_it_through():