Code formatting.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/976

NEWS

@@ -1,3 +1,44 @@
+1.9.7
+ * #855: Add a Sentry monitoring hook. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook
+ * #968: Fix for a "spot" check error when a filename in the most recent archive contains a newline.
+ * #970: Fix for an error when there's a blank line in the configured patterns or excludes.
+ * #971: Fix for "exclude_from" files being completely ignored.
+ * #977: Fix for "exclude_patterns" and "exclude_from" not supporting explicit pattern styles (e.g.,
+   "sh:" or "re:").
+
+1.9.6
+ * #959: Fix an error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
+   source directories.
+ * #960: Fix for archives storing relative source directory paths such that they contain the working
+   directory.
+ * #960: Fix the "spot" check to support relative source directory paths.
+ * #962: For the ZFS, Btrfs, and LVM hooks, perform path rewriting for excludes and patterns in
+   addition to the existing source directories rewriting.
+ * #962: Under the hood, merge all configured source directories, excludes, and patterns into a
+   unified temporary patterns file for passing to Borg. The borgmatic configuration options remain
+   unchanged.
+ * #962: For the LVM hook, add support for nested logical volumes.
+ * #965: Fix a borgmatic runtime directory error when running the "spot" check with a database hook
+   enabled.
+ * #969: Fix the "restore" action to work on database dumps without a port when a default port is
+   present in configuration.
+ * Fix the "spot" check to no longer consider pipe files within an archive for file comparisons.
+ * Fix the "spot" check to have a nicer error when there are no source paths to compare.
+ * Fix auto-excluding of special files (when databases are configured) to support relative source
+   directory paths.
+ * Drop support for Python 3.8, which has been end-of-lifed.
+
+1.9.5
+ * #418: Backup and restore databases that have the same name but with different ports, hostnames,
+   or hooks.
+ * #947: To avoid a hang in the database hooks, error and exit when the borgmatic runtime
+   directory overlaps with the configured excludes.
+ * #954: Fix a findmnt command error in the Btrfs hook by switching to parsing JSON output.
+ * #956: Fix the printing of a color reset code even when color is disabled.
+ * #958: Drop colorama as a library dependency.
+ * When the ZFS, Btrfs, or LVM hooks aren't configured, don't try to cleanup snapshots for them.
+
 1.9.4
  * #80 (beta): Add an LVM hook for snapshotting and backing up LVM logical volumes. See the
    documentation for more information:

README.md

@@ -75,6 +75,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 

borgmatic/actions/check.py

@@ -367,14 +367,16 @@ def collect_spot_check_source_paths(
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
         ).values()
     )
+    working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, pattern_file, exclude_file) = (
+    (create_flags, create_positional_arguments, pattern_file) = (
         borgmatic.borg.create.make_base_create_command(
             dry_run=True,
             repository_path=repository['path'],
             config=config,
-            source_directories=borgmatic.actions.create.process_source_directories(
-                config,
+            patterns=borgmatic.actions.create.process_patterns(
+                borgmatic.actions.create.collect_patterns(config),
+                working_directory,
             ),
             local_borg_version=local_borg_version,
             global_arguments=global_arguments,
@@ -399,7 +401,7 @@ def collect_spot_check_source_paths(
 
     paths = tuple(
         path_line.split(' ', 1)[1]
-        for path_line in paths_output.split('\n')
+        for path_line in paths_output.splitlines()
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
 
@@ -409,6 +411,7 @@ def collect_spot_check_source_paths(
 
 
 BORG_DIRECTORY_FILE_TYPE = 'd'
+BORG_PIPE_FILE_TYPE = 'p'
 
 
 def collect_spot_check_archive_paths(
@@ -426,6 +429,9 @@ def collect_spot_check_archive_paths(
     local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the
     remote Borg path, and the borgmatic runtime directory, collect the paths from the given archive
     (but only include files and symlinks and exclude borgmatic runtime directories).
+
+    These paths do not have a leading slash, as that's how Borg stores them. As a result, we don't
+    know whether they came from absolute or relative source directories.
     '''
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
@@ -437,15 +443,17 @@ def collect_spot_check_archive_paths(
             config,
             local_borg_version,
             global_arguments,
-            path_format='{type} /{path}{NL}',  # noqa: FS003
+            path_format='{type} {path}{NL}',  # noqa: FS003
             local_path=local_path,
             remote_path=remote_path,
         )
         for (file_type, path) in (line.split(' ', 1),)
-        if file_type != BORG_DIRECTORY_FILE_TYPE
-        if pathlib.Path('/borgmatic') not in pathlib.Path(path).parents
-        if pathlib.Path(borgmatic_source_directory) not in pathlib.Path(path).parents
-        if pathlib.Path(borgmatic_runtime_directory) not in pathlib.Path(path).parents
+        if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
+        if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
+        if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
+        not in pathlib.Path(path).parents
+        if pathlib.Path(borgmatic_runtime_directory.lstrip(os.path.sep))
+        not in pathlib.Path(path).parents
     )
 
 
@@ -532,7 +540,7 @@ def compare_spot_check_hashes(
                     local_borg_version,
                     global_arguments,
                     list_paths=source_sample_paths_subset,
-                    path_format='{xxh64} /{path}{NL}',  # noqa: FS003
+                    path_format='{xxh64} {path}{NL}',  # noqa: FS003
                     local_path=local_path,
                     remote_path=remote_path,
                 )
@@ -544,7 +552,7 @@ def compare_spot_check_hashes(
     failing_paths = []
 
     for path, source_hash in source_hashes.items():
-        archive_hash = archive_hashes.get(path)
+        archive_hash = archive_hashes.get(path.lstrip(os.path.sep))
 
         if archive_hash is not None and archive_hash == source_hash:
             continue
@@ -621,16 +629,25 @@ def spot_check(
     )
     logger.debug(f'{log_prefix}: {len(archive_paths)} total archive paths for spot check')
 
+    if len(source_paths) == 0:
+        logger.debug(
+            f'{log_prefix}: Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}'
+        )
+        raise ValueError(
+            'Spot check failed: There are no source paths to compare against the archive'
+        )
+
     # Calculate the percentage delta between the source paths count and the archive paths count, and
     # compare that delta to the configured count tolerance percentage.
     count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
 
     if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
+        rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
         logger.debug(
-            f'{log_prefix}: Paths in source paths but not latest archive: {", ".join(set(source_paths) - set(archive_paths)) or "none"}'
+            f'{log_prefix}: Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}'
         )
         logger.debug(
-            f'{log_prefix}: Paths in latest archive but not source paths: {", ".join(set(archive_paths) - set(source_paths)) or "none"}'
+            f'{log_prefix}: Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}'
         )
         raise ValueError(
             f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'

borgmatic/actions/config/bootstrap.py

@@ -41,7 +41,7 @@ def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_
     config = make_bootstrap_config(bootstrap_arguments)
 
     # Probe for the manifest file in multiple locations, as the default location has moved to the
-    # borgmatic runtime directory (which get stored as just "/borgmatic" with Borg 1.4+). But we
+    # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
     # still want to support reading the manifest from previously created archives as well.
     with borgmatic.config.paths.Runtime_directory(
         {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},

borgmatic/actions/create.py

@@ -6,6 +6,7 @@ import pathlib
 
 import borgmatic.actions.json
 import borgmatic.borg.create
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.config.validate
 import borgmatic.hooks.command
@@ -14,132 +15,239 @@ import borgmatic.hooks.dispatch
 logger = logging.getLogger(__name__)
 
 
+def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
+    '''
+    Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
+    return it.
+    '''
+    try:
+        (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
+    except ValueError:
+        raise ValueError(f'Invalid pattern: {pattern_line}')
+
+    try:
+        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
+        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
+    except ValueError:
+        pattern_style = default_style
+        path = remainder
+
+    return borgmatic.borg.pattern.Pattern(
+        path,
+        borgmatic.borg.pattern.Pattern_type(pattern_type),
+        borgmatic.borg.pattern.Pattern_style(pattern_style),
+    )
+
+
+def collect_patterns(config):
+    '''
+    Given a configuration dict, produce a single sequence of patterns comprised of the configured
+    source directories, patterns, excludes, pattern files, and exclude files.
+
+    The idea is that Borg has all these different ways of specifying includes, excludes, source
+    directories, etc., but we'd like to collapse them all down to one common format (patterns) for
+    ease of manipulation within borgmatic.
+    '''
+    try:
+        return (
+            tuple(
+                borgmatic.borg.pattern.Pattern(source_directory)
+                for source_directory in config.get('source_directories', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip(), borgmatic.borg.pattern.Pattern_style.SHELL)
+                for pattern_line in config.get('patterns', ())
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for exclude_line in config.get('exclude_patterns', ())
+            )
+            + tuple(
+                parse_pattern(pattern_line.strip(), borgmatic.borg.pattern.Pattern_style.SHELL)
+                for filename in config.get('patterns_from', ())
+                for pattern_line in open(filename).readlines()
+                if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
+            )
+            + tuple(
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                )
+                for filename in config.get('exclude_from', ())
+                for exclude_line in open(filename).readlines()
+                if not exclude_line.lstrip().startswith('#')
+                if exclude_line.strip()
+            )
+        )
+    except (FileNotFoundError, OSError) as error:
+        logger.debug(error)
+
+        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
+
+
 def expand_directory(directory, working_directory):
     '''
     Given a directory path, expand any tilde (representing a user's home directory) and any globs
     therein. Return a list of one or more resulting paths.
-    '''
-    expanded_directory = os.path.join(working_directory or '', os.path.expanduser(directory))
 
-    return glob.glob(expanded_directory) or [expanded_directory]
+    Take into account the given working directory so that relative paths are supported.
+    '''
+    expanded_directory = os.path.expanduser(directory)
+
+    # This would be a lot easier to do with glob(..., root_dir=working_directory), but root_dir is
+    # only available in Python 3.10+.
+    normalized_directory = os.path.join(working_directory or '', expanded_directory)
+    glob_paths = glob.glob(normalized_directory)
+
+    if not glob_paths:
+        return [expanded_directory]
+
+    working_directory_prefix = os.path.join(working_directory or '', '')
+
+    return [
+        (
+            glob_path
+            # If these are equal, that means we didn't add any working directory prefix above.
+            if normalized_directory == expanded_directory
+            # Remove the working directory prefix that we added above in order to make glob() work.
+            # We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
+            # hack.
+            else glob_path.removeprefix(working_directory_prefix)
+        )
+        for glob_path in glob_paths
+    ]
 
 
-def expand_directories(directories, working_directory=None):
+def expand_patterns(patterns, working_directory=None, skip_paths=None):
     '''
-    Given a sequence of directory paths and an optional working directory, expand tildes and globs
-    in each one. Return all the resulting directories as a single flattened tuple.
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    expand tildes and globs in each root pattern. Return all the resulting patterns (not just the
+    root patterns) as a tuple.
+
+    If a set of paths are given to skip, then don't expand any patterns matching them.
     '''
-    if directories is None:
+    if patterns is None:
         return ()
 
     return tuple(
         itertools.chain.from_iterable(
-            expand_directory(directory, working_directory) for directory in directories
+            (
+                (
+                    borgmatic.borg.pattern.Pattern(
+                        expanded_path,
+                        pattern.type,
+                        pattern.style,
+                        pattern.device,
+                    )
+                    for expanded_path in expand_directory(pattern.path, working_directory)
+                )
+                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and pattern.path not in (skip_paths or ())
+                else (pattern,)
+            )
+            for pattern in patterns
         )
     )
 
 
-def map_directories_to_devices(directories, working_directory=None):
+def device_map_patterns(patterns, working_directory=None):
     '''
-    Given a sequence of directories and an optional working directory, return a map from directory
-    to an identifier for the device on which that directory resides or None if the path doesn't
-    exist.
+    Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory,
+    determine the identifier for the device on which the pattern's path resides—or None if the path
+    doesn't exist or is from a non-root pattern. Return an updated sequence of patterns with the
+    device field populated. But if the device field is already set, don't bother setting it again.
 
-    This is handy for determining whether two different directories are on the same filesystem (have
-    the same device identifier).
+    This is handy for determining whether two different pattern paths are on the same filesystem
+    (have the same device identifier).
     '''
-    return {
-        directory: os.stat(full_directory).st_dev if os.path.exists(full_directory) else None
-        for directory in directories
-        for full_directory in (os.path.join(working_directory or '', directory),)
-    }
+    return tuple(
+        borgmatic.borg.pattern.Pattern(
+            pattern.path,
+            pattern.type,
+            pattern.style,
+            device=pattern.device
+            or (
+                os.stat(full_path).st_dev
+                if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+                and os.path.exists(full_path)
+                else None
+            ),
+        )
+        for pattern in patterns
+        for full_path in (os.path.join(working_directory or '', pattern.path),)
+    )
 
 
-def deduplicate_directories(directory_devices, additional_directory_devices):
+def deduplicate_patterns(patterns):
     '''
-    Given a map from directory to the identifier for the device on which that directory resides,
-    return the directories as a sorted sequence with all duplicate child directories removed. For
-    instance, if paths is ['/foo', '/foo/bar'], return just: ['/foo']
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
+    root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
+    "/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
+    modification.
 
-    The one exception to this rule is if two paths are on different filesystems (devices). In that
-    case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
-    location.one_file_system option is true).
+    The one exception to deduplication is two paths are on different filesystems (devices). In that
+    case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
+    one_file_system option is true).
 
-    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
-    child directories, because it will naturally spider the contents of the parent directory. And
+    The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
+    child patterns, because it will naturally spider the contents of the parent pattern's path. And
     there are cases where Borg coming across the same file twice will result in duplicate reads and
     even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
     Borg.
-
-    If any additional directory devices are given, also deduplicate against them, but don't include
-    them in the returned directories.
     '''
-    deduplicated = set()
-    directories = sorted(directory_devices.keys())
-    additional_directories = sorted(additional_directory_devices.keys())
-    all_devices = {**directory_devices, **additional_directory_devices}
+    deduplicated = {}  # Use just the keys as an ordered set.
 
-    for directory in directories:
-        deduplicated.add(directory)
-        parents = pathlib.PurePath(directory).parents
+    for pattern in patterns:
+        if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+            deduplicated[pattern] = True
+            continue
 
-        # If another directory in the given list (or the additional list) is a parent of current
-        # directory (even n levels up) and both are on the same filesystem, then the current
-        # directory is a duplicate.
-        for other_directory in directories + additional_directories:
-            for parent in parents:
-                if (
-                    pathlib.PurePath(other_directory) == parent
-                    and all_devices[directory] is not None
-                    and all_devices[other_directory] == all_devices[directory]
-                ):
-                    if directory in deduplicated:
-                        deduplicated.remove(directory)
-                    break
+        parents = pathlib.PurePath(pattern.path).parents
 
-    return sorted(deduplicated)
+        # If another directory in the given list is a parent of current directory (even n levels up)
+        # and both are on the same filesystem, then the current directory is a duplicate.
+        for other_pattern in patterns:
+            if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
+                continue
+
+            if any(
+                pathlib.PurePath(other_pattern.path) == parent
+                and pattern.device is not None
+                and other_pattern.device == pattern.device
+                for parent in parents
+            ):
+                break
+        else:
+            deduplicated[pattern] = True
+
+    return tuple(deduplicated.keys())
 
 
-ROOT_PATTERN_PREFIX = 'R '
-
-
-def pattern_root_directories(patterns=None):
+def process_patterns(patterns, working_directory, skip_expand_paths=None):
     '''
-    Given a sequence of patterns, parse out and return just the root directories.
+    Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
+    "root" patterns, returning the resulting root and non-root patterns as a list.
+
+    If any paths are given to skip, don't expand them.
     '''
-    if not patterns:
-        return []
+    skip_paths = set(skip_expand_paths or ())
 
-    return [
-        pattern.split(ROOT_PATTERN_PREFIX, maxsplit=1)[1]
-        for pattern in patterns
-        if pattern.startswith(ROOT_PATTERN_PREFIX)
-    ]
-
-
-def process_source_directories(config, source_directories=None):
-    '''
-    Given a sequence of source directories (either in the source_directories argument or, lacking
-    that, from config), expand and deduplicate the source directories, returning the result.
-    '''
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
-    if source_directories is None:
-        source_directories = tuple(config.get('source_directories', ()))
-
-    return deduplicate_directories(
-        map_directories_to_devices(
-            expand_directories(
-                tuple(source_directories),
-                working_directory=working_directory,
+    return list(
+        deduplicate_patterns(
+            device_map_patterns(
+                expand_patterns(
+                    patterns,
+                    working_directory=working_directory,
+                    skip_paths=skip_paths,
+                )
             )
-        ),
-        additional_directory_devices=map_directories_to_devices(
-            expand_directories(
-                pattern_root_directories(config.get('patterns')),
-                working_directory=working_directory,
-            )
-        ),
+        )
     )
 
 
@@ -177,6 +285,7 @@ def run_create(
 
     log_prefix = repository.get('label', repository['path'])
     logger.info(f'{log_prefix}: Creating archive{dry_run_label}')
+    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     with borgmatic.config.paths.Runtime_directory(
         config, log_prefix
@@ -189,7 +298,7 @@ def run_create(
             borgmatic_runtime_directory,
             global_arguments.dry_run,
         )
-        source_directories = process_source_directories(config)
+        patterns = process_patterns(collect_patterns(config), working_directory)
         active_dumps = borgmatic.hooks.dispatch.call_hooks(
             'dump_data_sources',
             config,
@@ -197,21 +306,21 @@ def run_create(
             borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
             config_paths,
             borgmatic_runtime_directory,
-            source_directories,
+            patterns,
             global_arguments.dry_run,
         )
 
-        # Process source directories again in case any data source hooks updated them. Without this
-        # step, we could end up with duplicate paths that cause Borg to hang when it tries to read
-        # from the same named pipe twice.
-        source_directories = process_source_directories(config, source_directories)
+        # Process the patterns again in case any data source hooks updated them. Without this step,
+        # we could end up with duplicate paths that cause Borg to hang when it tries to read from
+        # the same named pipe twice.
+        patterns = process_patterns(patterns, working_directory, skip_expand_paths=config_paths)
         stream_processes = [process for processes in active_dumps.values() for process in processes]
 
         json_output = borgmatic.borg.create.create_archive(
             global_arguments.dry_run,
             repository['path'],
             config,
-            source_directories,
+            patterns,
             local_borg_version,
             global_arguments,
             borgmatic_runtime_directory,

borgmatic/actions/restore.py

@@ -1,4 +1,4 @@
-import copy
+import collections
 import logging
 import os
 import pathlib
@@ -17,53 +17,106 @@ import borgmatic.hooks.dispatch
 logger = logging.getLogger(__name__)
 
 
-UNSPECIFIED_HOOK = object()
+UNSPECIFIED = object()
 
 
-def get_configured_data_source(
-    config,
-    archive_data_source_names,
-    hook_name,
-    data_source_name,
-    configuration_data_source_name=None,
-):
+Dump = collections.namedtuple(
+    'Dump',
+    ('hook_name', 'data_source_name', 'hostname', 'port'),
+    defaults=('localhost', None),
+)
+
+
+def dumps_match(first, second, default_port=None):
     '''
-    Find the first data source with the given hook name and data source name in the configuration
-    dict and the given archive data source names dict (from hook name to data source names contained
-    in a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all data
-    source hooks for the named data source. If a configuration data source name is given, use that
-    instead of the data source name to lookup the data source in the given hooks configuration.
-
-    Return the found data source as a tuple of (found hook name, data source configuration dict) or
-    (None, None) if not found.
+    Compare two Dump instances for equality while supporting a field value of UNSPECIFIED, which
+    indicates that the field should match any value. If a default port is given, then consider any
+    dump having that port to match with a dump having a None port.
     '''
-    if not configuration_data_source_name:
-        configuration_data_source_name = data_source_name
+    for field_name in first._fields:
+        first_value = getattr(first, field_name)
+        second_value = getattr(second, field_name)
 
-    if hook_name == UNSPECIFIED_HOOK:
-        hooks_to_search = {
-            hook_name: value
-            for (hook_name, value) in config.items()
-            if hook_name.split('_databases')[0]
-            in borgmatic.hooks.dispatch.get_submodule_names(borgmatic.hooks.data_source)
-        }
+        if default_port is not None and field_name == 'port':
+            if first_value == default_port and second_value is None:
+                continue
+            if second_value == default_port and first_value is None:
+                continue
+
+        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
+            continue
+
+        if first_value != second_value:
+            return False
+
+    return True
+
+
+def render_dump_metadata(dump):
+    '''
+    Given a Dump instance, make a display string describing it for use in log messages.
+    '''
+    name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
+    hostname = dump.hostname or 'localhost'
+    port = None if dump.port is UNSPECIFIED else dump.port
+
+    if port:
+        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
     else:
-        try:
-            hooks_to_search = {hook_name: config[hook_name]}
-        except KeyError:
-            return (None, None)
+        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
 
-    return next(
-        (
-            (name, hook_data_source)
-            for (name, hook) in hooks_to_search.items()
-            for hook_data_source in hook
-            if hook_data_source['name'] == configuration_data_source_name
-            and data_source_name in archive_data_source_names.get(name, [])
-        ),
-        (None, None),
+    if dump.hook_name not in (None, UNSPECIFIED):
+        return f'{metadata} ({dump.hook_name})'
+
+    return metadata
+
+
+def get_configured_data_source(config, restore_dump, log_prefix):
+    '''
+    Search in the given configuration dict for dumps corresponding to the given dump to restore. If
+    there are multiple matches, error. Log using the given log prefix.
+
+    Return the found data source as a data source configuration dict or None if not found.
+    '''
+    try:
+        hooks_to_search = {restore_dump.hook_name: config[restore_dump.hook_name]}
+    except KeyError:
+        return None
+
+    matching_dumps = tuple(
+        hook_data_source
+        for (hook_name, hook_config) in hooks_to_search.items()
+        for hook_data_source in hook_config
+        for default_port in (
+            borgmatic.hooks.dispatch.call_hook(
+                function_name='get_default_port',
+                config=config,
+                log_prefix=log_prefix,
+                hook_name=hook_name,
+            ),
+        )
+        if dumps_match(
+            Dump(
+                hook_name,
+                hook_data_source.get('name'),
+                hook_data_source.get('hostname', 'localhost'),
+                hook_data_source.get('port'),
+            ),
+            restore_dump,
+            default_port,
+        )
     )
 
+    if not matching_dumps:
+        return None
+
+    if len(matching_dumps) > 1:
+        raise ValueError(
+            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
+        )
+
+    return matching_dumps[0]
+
 
 def strip_path_prefix_from_extracted_dump_destination(
     destination_path, borgmatic_runtime_directory
@@ -98,7 +151,7 @@ def strip_path_prefix_from_extracted_dump_destination(
         break
 
 
-def restore_single_data_source(
+def restore_single_dump(
     repository,
     config,
     local_borg_version,
@@ -116,8 +169,12 @@ def restore_single_data_source(
     username/password as connection params, and a configured data source configuration dict, restore
     that data source from the archive.
     '''
+    dump_metadata = render_dump_metadata(
+        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
+    )
+
     logger.info(
-        f'{repository.get("label", repository["path"])}: Restoring data source {data_source["name"]}'
+        f'{repository.get("label", repository["path"])}: Restoring data source {dump_metadata}'
     )
 
     dump_patterns = borgmatic.hooks.dispatch.call_hooks(
@@ -127,7 +184,7 @@ def restore_single_data_source(
         borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
         borgmatic_runtime_directory,
         data_source['name'],
-    )[hook_name.split('_databases')[0]]
+    )[hook_name.split('_databases', 1)[0]]
 
     destination_path = (
         tempfile.mkdtemp(dir=borgmatic_runtime_directory)
@@ -180,7 +237,7 @@ def restore_single_data_source(
     )
 
 
-def collect_archive_data_source_names(
+def collect_dumps_from_archive(
     repository,
     archive,
     config,
@@ -192,17 +249,17 @@ def collect_archive_data_source_names(
 ):
     '''
     Given a local or remote repository path, a resolved archive name, a configuration dict, the
-    local Borg version, global_arguments an argparse.Namespace, local and remote Borg paths, and the
-    borgmatic runtime directory, query the archive for the names of data sources it contains as
-    dumps and return them as a dict from hook name to a sequence of data source names.
+    local Borg version, global arguments an argparse.Namespace, local and remote Borg paths, and the
+    borgmatic runtime directory, query the archive for the names of data sources dumps it contains
+    and return them as a set of Dump instances.
     '''
     borgmatic_source_directory = str(
         pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
     )
 
     # Probe for the data source dumps in multiple locations, as the default location has moved to
-    # the borgmatic runtime directory (which get stored as just "/borgmatic" with Borg 1.4+). But we
-    # still want to support reading dumps from previously created archives as well.
+    # the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
+    # we still want to support reading dumps from previously created archives as well.
     dump_paths = borgmatic.borg.list.capture_archive_listing(
         repository,
         archive,
@@ -224,110 +281,145 @@ def collect_archive_data_source_names(
         remote_path=remote_path,
     )
 
-    # Determine the data source names corresponding to the dumps found in the archive and
-    # add them to restore_names.
-    archive_data_source_names = {}
+    # Parse the paths of dumps found in the archive to get their respective dump metadata.
+    dumps_from_archive = set()
 
     for dump_path in dump_paths:
         if not dump_path:
             continue
 
+        # Probe to find the base directory that's at the start of the dump path.
         for base_directory in (
             'borgmatic',
             borgmatic_runtime_directory,
             borgmatic_source_directory,
         ):
             try:
-                (hook_name, _, data_source_name) = dump_path.split(base_directory + os.path.sep, 1)[
-                    1
-                ].split(os.path.sep)[0:3]
+                (hook_name, host_and_port, data_source_name) = dump_path.split(
+                    base_directory + os.path.sep, 1
+                )[1].split(os.path.sep)[0:3]
             except (ValueError, IndexError):
-                pass
-            else:
-                if data_source_name not in archive_data_source_names.get(hook_name, []):
-                    archive_data_source_names.setdefault(hook_name, []).extend([data_source_name])
-                    break
+                continue
+
+            parts = host_and_port.split(':', 1)
+
+            if len(parts) == 1:
+                parts += (None,)
+
+            (hostname, port) = parts
+
+            try:
+                port = int(port)
+            except (ValueError, TypeError):
+                port = None
+
+            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
+
+            # We've successfully parsed the dump path, so need to probe any further.
+            break
         else:
             logger.warning(
                 f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
             )
 
-    return archive_data_source_names
+    return dumps_from_archive
 
 
-def find_data_sources_to_restore(requested_data_source_names, archive_data_source_names):
+def get_dumps_to_restore(restore_arguments, dumps_from_archive):
     '''
-    Given a sequence of requested data source names to restore and a dict of hook name to the names
-    of data sources found in an archive, return an expanded sequence of data source names to
-    restore, replacing "all" with actual data source names as appropriate.
+    Given restore arguments as an argparse.Namespace instance indicating which dumps to restore and
+    a set of Dump instances representing the dumps found in an archive, return a set of specific
+    Dump instances from the archive to restore. As part of this, replace any Dump having a data
+    source name of "all" with multiple named Dump instances as appropriate.
 
-    Raise ValueError if any of the requested data source names cannot be found in the archive.
+    Raise ValueError if any of the requested data source names cannot be found in the archive or if
+    there are multiple archive dump matches for a given requested dump.
     '''
-    # A map from data source hook name to the data source names to restore for that hook.
-    restore_names = (
-        {UNSPECIFIED_HOOK: requested_data_source_names}
-        if requested_data_source_names
-        else {UNSPECIFIED_HOOK: ['all']}
+    requested_dumps = (
+        {
+            Dump(
+                hook_name=(
+                    (
+                        restore_arguments.hook
+                        if restore_arguments.hook.endswith('_databases')
+                        else f'{restore_arguments.hook}_databases'
+                    )
+                    if restore_arguments.hook
+                    else UNSPECIFIED
+                ),
+                data_source_name=name,
+                hostname=restore_arguments.original_hostname or 'localhost',
+                port=restore_arguments.original_port,
+            )
+            for name in restore_arguments.data_sources
+        }
+        if restore_arguments.data_sources
+        else {
+            Dump(
+                hook_name=UNSPECIFIED,
+                data_source_name='all',
+                hostname=UNSPECIFIED,
+                port=UNSPECIFIED,
+            )
+        }
     )
+    missing_dumps = set()
+    dumps_to_restore = set()
 
-    # If "all" is in restore_names, then replace it with the names of dumps found within the
-    # archive.
-    if 'all' in restore_names[UNSPECIFIED_HOOK]:
-        restore_names[UNSPECIFIED_HOOK].remove('all')
+    # If there's a requested "all" dump, add every dump from the archive to the dumps to restore.
+    if any(dump for dump in requested_dumps if dump.data_source_name == 'all'):
+        dumps_to_restore.update(dumps_from_archive)
 
-        for hook_name, data_source_names in archive_data_source_names.items():
-            restore_names.setdefault(hook_name, []).extend(data_source_names)
+    # If any archive dump matches a requested dump, add the archive dump to the dumps to restore.
+    for requested_dump in requested_dumps:
+        if requested_dump.data_source_name == 'all':
+            continue
 
-            # If a data source is to be restored as part of "all", then remove it from restore names
-            # so it doesn't get restored twice.
-            for data_source_name in data_source_names:
-                if data_source_name in restore_names[UNSPECIFIED_HOOK]:
-                    restore_names[UNSPECIFIED_HOOK].remove(data_source_name)
-
-    if not restore_names[UNSPECIFIED_HOOK]:
-        restore_names.pop(UNSPECIFIED_HOOK)
-
-    combined_restore_names = set(
-        name for data_source_names in restore_names.values() for name in data_source_names
-    )
-    combined_archive_data_source_names = set(
-        name
-        for data_source_names in archive_data_source_names.values()
-        for name in data_source_names
-    )
-
-    missing_names = sorted(set(combined_restore_names) - combined_archive_data_source_names)
-    if missing_names:
-        joined_names = ', '.join(f'"{name}"' for name in missing_names)
-        raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from archive"
+        matching_dumps = tuple(
+            archive_dump
+            for archive_dump in dumps_from_archive
+            if dumps_match(requested_dump, archive_dump)
         )
 
-    return restore_names
+        if len(matching_dumps) == 0:
+            missing_dumps.add(requested_dump)
+        elif len(matching_dumps) == 1:
+            dumps_to_restore.add(matching_dumps[0])
+        else:
+            raise ValueError(
+                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
+            )
+
+    if missing_dumps:
+        rendered_dumps = ', '.join(
+            f'{render_dump_metadata(dump)}' for dump in sorted(missing_dumps)
+        )
+
+        raise ValueError(
+            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
+        )
+
+    return dumps_to_restore
 
 
-def ensure_data_sources_found(restore_names, remaining_restore_names, found_names):
+def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
     '''
-    Given a dict from hook name to data source names to restore, a dict from hook name to remaining
-    data source names to restore, and a sequence of found (actually restored) data source names,
-    raise ValueError if requested data source to restore were missing from the archive and/or
+    Given a set of requested dumps to restore and a set of dumps actually restored, raise ValueError
+    if any requested dumps to restore weren't restored, indicating that they were missing from the
     configuration.
     '''
-    combined_restore_names = set(
-        name
-        for data_source_names in tuple(restore_names.values())
-        + tuple(remaining_restore_names.values())
-        for name in data_source_names
-    )
-
-    if not combined_restore_names and not found_names:
+    if not dumps_actually_restored:
         raise ValueError('No data source dumps were found to restore')
 
-    missing_names = sorted(set(combined_restore_names) - set(found_names))
-    if missing_names:
-        joined_names = ', '.join(f'"{name}"' for name in missing_names)
+    missing_dumps = sorted(
+        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
+    )
+
+    if missing_dumps:
+        rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
+
         raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from borgmatic's configuration"
+            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
         )
 
 
@@ -344,7 +436,8 @@ def run_restore(
     Run the "restore" action for the given repository, but only if the repository matches the
     requested repository in restore arguments.
 
-    Raise ValueError if a configured data source could not be found to restore.
+    Raise ValueError if a configured data source could not be found to restore or there's no
+    matching dump in the archive.
     '''
     if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
         repository, restore_arguments.repository
@@ -375,7 +468,7 @@ def run_restore(
             local_path,
             remote_path,
         )
-        archive_data_source_names = collect_archive_data_source_names(
+        dumps_from_archive = collect_dumps_from_archive(
             repository['path'],
             archive_name,
             config,
@@ -385,11 +478,9 @@ def run_restore(
             remote_path,
             borgmatic_runtime_directory,
         )
-        restore_names = find_data_sources_to_restore(
-            restore_arguments.data_sources, archive_data_source_names
-        )
-        found_names = set()
-        remaining_restore_names = {}
+        dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
+
+        dumps_actually_restored = set()
         connection_params = {
             'hostname': restore_arguments.hostname,
             'port': restore_arguments.port,
@@ -398,61 +489,44 @@ def run_restore(
             'restore_path': restore_arguments.restore_path,
         }
 
-        for hook_name, data_source_names in restore_names.items():
-            for data_source_name in data_source_names:
-                found_hook_name, found_data_source = get_configured_data_source(
-                    config, archive_data_source_names, hook_name, data_source_name
-                )
+        # Restore each dump.
+        for restore_dump in dumps_to_restore:
+            found_data_source = get_configured_data_source(
+                config,
+                restore_dump,
+                log_prefix=repository['path'],
+            )
 
-                if not found_data_source:
-                    remaining_restore_names.setdefault(found_hook_name or hook_name, []).append(
-                        data_source_name
-                    )
-                    continue
-
-                found_names.add(data_source_name)
-                restore_single_data_source(
-                    repository,
+            # For a dump that wasn't found via an exact match in the configuration, try to fallback
+            # to an "all" data source.
+            if not found_data_source:
+                found_data_source = get_configured_data_source(
                     config,
-                    local_borg_version,
-                    global_arguments,
-                    local_path,
-                    remote_path,
-                    archive_name,
-                    found_hook_name or hook_name,
-                    dict(found_data_source, **{'schemas': restore_arguments.schemas}),
-                    connection_params,
-                    borgmatic_runtime_directory,
-                )
-
-        # For any data sources that weren't found via exact matches in the configuration, try to
-        # fallback to "all" entries.
-        for hook_name, data_source_names in remaining_restore_names.items():
-            for data_source_name in data_source_names:
-                found_hook_name, found_data_source = get_configured_data_source(
-                    config, archive_data_source_names, hook_name, data_source_name, 'all'
+                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
+                    log_prefix=repository['path'],
                 )
 
                 if not found_data_source:
                     continue
 
-                found_names.add(data_source_name)
-                data_source = copy.copy(found_data_source)
-                data_source['name'] = data_source_name
+                found_data_source = dict(found_data_source)
+                found_data_source['name'] = restore_dump.data_source_name
 
-                restore_single_data_source(
-                    repository,
-                    config,
-                    local_borg_version,
-                    global_arguments,
-                    local_path,
-                    remote_path,
-                    archive_name,
-                    found_hook_name or hook_name,
-                    dict(data_source, **{'schemas': restore_arguments.schemas}),
-                    connection_params,
-                    borgmatic_runtime_directory,
-                )
+            dumps_actually_restored.add(restore_dump)
+
+            restore_single_dump(
+                repository,
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path,
+                remote_path,
+                archive_name,
+                restore_dump.hook_name,
+                dict(found_data_source, **{'schemas': restore_arguments.schemas}),
+                connection_params,
+                borgmatic_runtime_directory,
+            )
 
         borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
             'remove_data_source_dumps',
@@ -463,4 +537,4 @@ def run_restore(
             global_arguments.dry_run,
         )
 
-    ensure_data_sources_found(restore_names, remaining_restore_names, found_names)
+    ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored)

borgmatic/borg/create.py

@@ -6,6 +6,7 @@ import stat
 import tempfile
 import textwrap
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.logger
 from borgmatic.borg import environment, feature, flags
@@ -19,81 +20,42 @@ from borgmatic.execute import (
 logger = logging.getLogger(__name__)
 
 
-def expand_home_directories(directories):
+def write_patterns_file(patterns, borgmatic_runtime_directory, log_prefix, patterns_file=None):
     '''
-    Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
-    Return the results as a tuple.
-    '''
-    if directories is None:
-        return ()
+    Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
+    temporary file in the given borgmatic runtime directory and return the file object so it can
+    continue to exist on disk as long as the caller needs it.
 
-    return tuple(os.path.expanduser(directory) for directory in directories)
+    Use the given log prefix in any logging.
 
-
-def write_pattern_file(patterns=None, sources=None, pattern_file=None):
-    '''
-    Given a sequence of patterns and an optional sequence of source directories, write them to a
-    named temporary file (with the source directories as additional roots) and return the file.
-    If an optional open pattern file is given, overwrite it instead of making a new temporary file.
+    If an optional open pattern file is given, append to it instead of making a new temporary file.
     Return None if no patterns are provided.
     '''
-    if not patterns and not sources:
+    if not patterns:
         return None
 
-    if pattern_file is None:
-        pattern_file = tempfile.NamedTemporaryFile('w')
+    if patterns_file is None:
+        patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
     else:
-        pattern_file.seek(0)
+        patterns_file.write('\n')
 
-    pattern_file.write(
-        '\n'.join(tuple(patterns or ()) + tuple(f'R {source}' for source in (sources or [])))
+    patterns_output = '\n'.join(
+        f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
+        for pattern in patterns
     )
-    pattern_file.flush()
+    logger.debug(f'{log_prefix}: Writing patterns to {patterns_file.name}:\n{patterns_output}')
 
-    return pattern_file
+    patterns_file.write(patterns_output)
+    patterns_file.flush()
+
+    return patterns_file
 
 
-def ensure_files_readable(*filename_lists):
+def make_exclude_flags(config):
     '''
-    Given a sequence of filename sequences, ensure that each filename is openable. This prevents
-    unreadable files from being passed to Borg, which in certain situations only warns instead of
-    erroring.
+    Given a configuration dict with various exclude options, return the corresponding Borg flags as
+    a tuple.
     '''
-    for file_object in itertools.chain.from_iterable(
-        filename_list for filename_list in filename_lists if filename_list
-    ):
-        open(file_object).close()
-
-
-def make_pattern_flags(config, pattern_filename=None):
-    '''
-    Given a configuration dict with a potential patterns_from option, and a filename containing any
-    additional patterns, return the corresponding Borg flags for those files as a tuple.
-    '''
-    pattern_filenames = tuple(config.get('patterns_from') or ()) + (
-        (pattern_filename,) if pattern_filename else ()
-    )
-
-    return tuple(
-        itertools.chain.from_iterable(
-            ('--patterns-from', pattern_filename) for pattern_filename in pattern_filenames
-        )
-    )
-
-
-def make_exclude_flags(config, exclude_filename=None):
-    '''
-    Given a configuration dict with various exclude options, and a filename containing any exclude
-    patterns, return the corresponding Borg flags as a tuple.
-    '''
-    exclude_filenames = tuple(config.get('exclude_from') or ()) + (
-        (exclude_filename,) if exclude_filename else ()
-    )
-    exclude_from_flags = tuple(
-        itertools.chain.from_iterable(
-            ('--exclude-from', exclude_filename) for exclude_filename in exclude_filenames
-        )
-    )
     caches_flag = ('--exclude-caches',) if config.get('exclude_caches') else ()
     if_present_flags = tuple(
         itertools.chain.from_iterable(
@@ -104,13 +66,7 @@ def make_exclude_flags(config, exclude_filename=None):
     keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
     exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
 
-    return (
-        exclude_from_flags
-        + caches_flag
-        + if_present_flags
-        + keep_exclude_tags_flags
-        + exclude_nodump_flags
-    )
+    return caches_flag + if_present_flags + keep_exclude_tags_flags + exclude_nodump_flags
 
 
 def make_list_filter_flags(local_borg_version, dry_run):
@@ -134,13 +90,14 @@ def make_list_filter_flags(local_borg_version, dry_run):
         return f'{base_flags}-'
 
 
-def special_file(path):
+def special_file(path, working_directory=None):
     '''
     Return whether the given path is a special file (character device, block device, or named pipe
-    / FIFO).
+    / FIFO). If a working directory is given, take it into account when making the full path to
+    check.
     '''
     try:
-        mode = os.stat(path).st_mode
+        mode = os.stat(os.path.join(working_directory or '', path)).st_mode
     except (FileNotFoundError, OSError):
         return False
 
@@ -160,14 +117,25 @@ def any_parent_directories(path, candidate_parents):
 
 
 def collect_special_file_paths(
-    create_command, config, local_path, working_directory, borg_environment, skip_directories
+    dry_run,
+    create_command,
+    config,
+    local_path,
+    working_directory,
+    borg_environment,
+    borgmatic_runtime_directory,
 ):
     '''
-    Given a Borg create command as a tuple, a configuration dict, a local Borg path, a working
-    directory, a dict of environment variables to pass to Borg, and a sequence of parent directories
-    to skip, collect the paths for any special files (character devices, block devices, and named
+    Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
+    a working directory, a dict of environment variables to pass to Borg, and the borgmatic runtime
+    directory, collect the paths for any special files (character devices, block devices, and named
     pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause
     Borg to hang if its --read-special flag is used.
+
+    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
+    its own special files there for database dumps. And if the borgmatic runtime directory is
+    configured to be excluded from the files Borg backs up, error, because this means Borg won't be
+    able to consume any database dumps and therefore borgmatic will hang.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
     # files including any named pipe we've created.
@@ -186,26 +154,39 @@ def collect_special_file_paths(
         for path_line in paths_output.split('\n')
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
+    skip_paths = {}
+
+    if os.path.exists(borgmatic_runtime_directory):
+        skip_paths = {
+            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
+        }
+
+        if not skip_paths and not dry_run:
+            raise ValueError(
+                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
+            )
 
     return tuple(
-        path
-        for path in paths
-        if special_file(path) and not any_parent_directories(path, skip_directories)
+        path for path in paths if special_file(path, working_directory) if path not in skip_paths
     )
 
 
-def check_all_source_directories_exist(source_directories):
+def check_all_root_patterns_exist(patterns):
     '''
-    Given a sequence of source directories, check that the source directories all exist. If any do
-    not, raise an exception.
+    Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
+    paths exist. If any don't, raise an exception.
     '''
-    missing_directories = [
-        source_directory
-        for source_directory in source_directories
-        if not os.path.exists(source_directory)
+    missing_paths = [
+        pattern.path
+        for pattern in patterns
+        if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
+        if not os.path.exists(pattern.path)
     ]
-    if missing_directories:
-        raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
+
+    if missing_paths:
+        raise ValueError(
+            f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}"
+        )
 
 
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
@@ -215,7 +196,7 @@ def make_base_create_command(
     dry_run,
     repository_path,
     config,
-    source_directories,
+    patterns,
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
@@ -227,23 +208,18 @@ def make_base_create_command(
     stream_processes=None,
 ):
     '''
-    Given vebosity/dry-run flags, a local or remote repository path, a configuration dict, a
-    sequence of loaded configuration paths, the local Borg version, global arguments as an
-    argparse.Namespace instance, and a sequence of borgmatic source directories, return a tuple of
-    (base Borg create command flags, Borg create command positional arguments, open pattern file
-    handle, open exclude file handle).
+    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
+    sequence of patterns as borgmatic.borg.pattern.Pattern instances, the local Borg version,
+    global arguments as an argparse.Namespace instance, and a sequence of borgmatic source
+    directories, return a tuple of (base Borg create command flags, Borg create command positional
+    arguments, open pattern file handle).
     '''
     if config.get('source_directories_must_exist', False):
-        check_all_source_directories_exist(source_directories)
+        check_all_root_patterns_exist(patterns)
 
-    ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
-
-    pattern_file = (
-        write_pattern_file(config.get('patterns'), source_directories)
-        if config.get('patterns') or config.get('patterns_from')
-        else None
+    patterns_file = write_patterns_file(
+        patterns, borgmatic_runtime_directory, log_prefix=repository_path
     )
-    exclude_file = write_pattern_file(expand_home_directories(config.get('exclude_patterns')))
     checkpoint_interval = config.get('checkpoint_interval', None)
     checkpoint_volume = config.get('checkpoint_volume', None)
     chunker_params = config.get('chunker_params', None)
@@ -286,8 +262,8 @@ def make_base_create_command(
     create_flags = (
         tuple(local_path.split(' '))
         + ('create',)
-        + make_pattern_flags(config, pattern_file.name if pattern_file else None)
-        + make_exclude_flags(config, exclude_file.name if exclude_file else None)
+        + (('--patterns-from', patterns_file.name) if patterns_file else ())
+        + make_exclude_flags(config)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
         + (('--checkpoint-volume', str(checkpoint_volume)) if checkpoint_volume else ())
         + (('--chunker-params', chunker_params) if chunker_params else ())
@@ -317,7 +293,7 @@ def make_base_create_command(
 
     create_positional_arguments = flags.make_repository_archive_flags(
         repository_path, archive_name_format, local_borg_version
-    ) + (tuple(source_directories) if not pattern_file else ())
+    )
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
@@ -330,14 +306,13 @@ def make_base_create_command(
 
         logger.debug(f'{repository_path}: Collecting special file paths')
         special_file_paths = collect_special_file_paths(
+            dry_run,
             create_flags + create_positional_arguments,
             config,
             local_path,
             working_directory,
             borg_environment,
-            skip_directories=(
-                [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else []
-            ),
+            borgmatic_runtime_directory=borgmatic_runtime_directory,
         )
 
         if special_file_paths:
@@ -349,22 +324,31 @@ def make_base_create_command(
             logger.warning(
                 f'{repository_path}: Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
             )
-            exclude_file = write_pattern_file(
-                expand_home_directories(
-                    tuple(config.get('exclude_patterns') or ()) + special_file_paths
+            patterns_file = write_patterns_file(
+                tuple(
+                    borgmatic.borg.pattern.Pattern(
+                        special_file_path,
+                        borgmatic.borg.pattern.Pattern_type.EXCLUDE,
+                        borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                    )
+                    for special_file_path in special_file_paths
                 ),
-                pattern_file=exclude_file,
+                borgmatic_runtime_directory,
+                log_prefix=repository_path,
+                patterns_file=patterns_file,
             )
-            create_flags += make_exclude_flags(config, exclude_file.name)
 
-    return (create_flags, create_positional_arguments, pattern_file, exclude_file)
+            if '--patterns-from' not in create_flags:
+                create_flags += ('--patterns-from', patterns_file.name)
+
+    return (create_flags, create_positional_arguments, patterns_file)
 
 
 def create_archive(
     dry_run,
     repository_path,
     config,
-    source_directories,
+    patterns,
     local_borg_version,
     global_arguments,
     borgmatic_runtime_directory,
@@ -377,7 +361,7 @@ def create_archive(
     stream_processes=None,
 ):
     '''
-    Given vebosity/dry-run flags, a local or remote repository path, a configuration dict, a
+    Given verbosity/dry-run flags, a local or remote repository path, a configuration dict, a
     sequence of loaded configuration paths, the local Borg version, and global arguments as an
     argparse.Namespace instance, create a Borg archive and return Borg's JSON output (if any).
 
@@ -388,22 +372,20 @@ def create_archive(
 
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    (create_flags, create_positional_arguments, pattern_file, exclude_file) = (
-        make_base_create_command(
-            dry_run,
-            repository_path,
-            config,
-            source_directories,
-            local_borg_version,
-            global_arguments,
-            borgmatic_runtime_directory,
-            local_path,
-            remote_path,
-            progress,
-            json,
-            list_files,
-            stream_processes,
-        )
+    (create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
+        dry_run,
+        repository_path,
+        config,
+        patterns,
+        local_borg_version,
+        global_arguments,
+        borgmatic_runtime_directory,
+        local_path,
+        remote_path,
+        progress,
+        json,
+        list_files,
+        stream_processes,
     )
 
     if json:

borgmatic/borg/list.py

@@ -120,7 +120,7 @@ def capture_archive_listing(
                     paths=[path for path in list_paths] if list_paths else None,
                     find_paths=None,
                     json=None,
-                    format=path_format or '{path}{NL}',  # noqa: FS003
+                    format=path_format or '{path}{NUL}',  # noqa: FS003
                 ),
                 global_arguments,
                 local_path,
@@ -132,7 +132,7 @@ def capture_archive_listing(
             borg_exit_codes=config.get('borg_exit_codes'),
         )
         .strip('\n')
-        .split('\n')
+        .split('\0')
     )
 
 
@@ -230,7 +230,7 @@ def list_archive(
                 borg_exit_codes=borg_exit_codes,
             )
             .strip('\n')
-            .split('\n')
+            .splitlines()
         )
     else:
         archive_lines = (list_arguments.archive,)

borgmatic/borg/pattern.py

@@ -0,0 +1,31 @@
+import collections
+import enum
+
+
+# See https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns
+class Pattern_type(enum.Enum):
+    ROOT = 'R'  # A ROOT pattern always has a NONE pattern style.
+    PATTERN_STYLE = 'P'
+    EXCLUDE = '-'
+    NO_RECURSE = '!'
+    INCLUDE = '+'
+
+
+class Pattern_style(enum.Enum):
+    NONE = ''
+    FNMATCH = 'fm'
+    SHELL = 'sh'
+    REGULAR_EXPRESSION = 're'
+    PATH_PREFIX = 'pp'
+    PATH_FULL_MATCH = 'pf'
+
+
+Pattern = collections.namedtuple(
+    'Pattern',
+    ('path', 'type', 'style', 'device'),
+    defaults=(
+        Pattern_type.ROOT,
+        Pattern_style.NONE,
+        None,
+    ),
+)

borgmatic/commands/arguments.py

@@ -1153,7 +1153,7 @@ def make_parsers():
         metavar='NAME',
         dest='data_sources',
         action='append',
-        help="Name of data source (e.g. database) to restore from archive, must be defined in borgmatic's configuration, can specify flag multiple times, defaults to all data sources in the archive",
+        help="Name of data source (e.g. database) to restore from the archive, must be defined in borgmatic's configuration, can specify the flag multiple times, defaults to all data sources in the archive",
     )
     restore_group.add_argument(
         '--schema',
@@ -1182,6 +1182,19 @@ def make_parsers():
         '--restore-path',
         help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--original-hostname',
+        help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
+    restore_group.add_argument(
+        '--original-port',
+        type=int,
+        help="The port where the dump to restore came from (if that port is in borgmatic's configuration), only necessary if you need to disambiguate dumps",
+    )
+    restore_group.add_argument(
+        '--hook',
+        help='The name of the data source hook for the dump to restore, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )

borgmatic/commands/borgmatic.py

@@ -8,8 +8,6 @@ import time
 from queue import Queue
 from subprocess import CalledProcessError
 
-import colorama
-
 import borgmatic.actions.borg
 import borgmatic.actions.break_lock
 import borgmatic.actions.change_passphrase
@@ -915,7 +913,7 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
     )
     color_enabled = should_do_markup(global_arguments.no_color or any_json_flags, configs)
-    colorama.init(autoreset=color_enabled, strip=not color_enabled)
+
     try:
         configure_logging(
             verbosity_to_log_level(global_arguments.verbosity),

borgmatic/config/schema.yaml

@@ -68,9 +68,7 @@ properties:
         type: boolean
         description: |
             Stay in same file system; do not cross mount points beyond the given
-            source directories. Defaults to false. But when a database hook is
-            used, the setting here is ignored and one_file_system is considered
-            true.
+            source directories. Defaults to false.
         example: true
     numeric_ids:
         type: boolean
@@ -133,8 +131,7 @@ properties:
             Any paths matching these patterns are included/excluded from
             backups. Globs are expanded. (Tildes are not.) See the output of
             "borg help patterns" for more details. Quote any value if it
-            contains leading punctuation, so it parses correctly. Note that only
-            one of "patterns" and "source_directories" may be used.
+            contains leading punctuation, so it parses correctly.
         example:
             - 'R /'
             - '- /home/*/.cache'
@@ -146,9 +143,8 @@ properties:
             type: string
         description: |
             Read include/exclude patterns from one or more separate named files,
-            one pattern per line. Note that Borg considers this option
-            experimental. See the output of "borg help patterns" for more
-            details.
+            one pattern per line. See the output of "borg help patterns" for
+            more details.
         example:
             - /etc/borgmatic/patterns
     exclude_patterns:
@@ -232,8 +228,8 @@ properties:
     source_directories_must_exist:
         type: boolean
         description: |
-            If true, then source directories must exist, otherwise an error is
-            raised. Defaults to false.
+            If true, then source directories (and root pattern paths) must
+            exist. If they don't, an error is raised. Defaults to false.
         example: true
     encryption_passcommand:
         type: string
@@ -963,8 +959,8 @@ properties:
                         dump all databases on the host. (Also set the "format"
                         to dump each database to a separate file instead of one
                         combined file.) Note that using this database hook
-                        implicitly enables both read_special and one_file_system
-                        (see above) to support dump and restore streaming.
+                        implicitly enables read_special (see above) to support
+                        dump and restore streaming.
                     example: users
                 hostname:
                     type: string
@@ -1145,9 +1141,8 @@ properties:
                     description: |
                         Database name (required if using this hook). Or "all" to
                         dump all databases on the host. Note that using this
-                        database hook implicitly enables both read_special and
-                        one_file_system (see above) to support dump and restore
-                        streaming.
+                        database hook implicitly enables read_special (see
+                        above) to support dump and restore streaming.
                     example: users
                 hostname:
                     type: string
@@ -1272,9 +1267,8 @@ properties:
                     description: |
                         Database name (required if using this hook). Or "all" to
                         dump all databases on the host. Note that using this
-                        database hook implicitly enables both read_special and
-                        one_file_system (see above) to support dump and restore
-                        streaming.
+                        database hook implicitly enables read_special (see
+                        above) to support dump and restore streaming.
                     example: users
                 hostname:
                     type: string
@@ -1407,9 +1401,9 @@ properties:
                     description: |
                         Path to the SQLite database file to dump. If relative,
                         it is relative to the current working directory. Note
-                        that using this database hook implicitly enables both
-                        read_special and one_file_system (see above) to support
-                        dump and restore streaming.
+                        that using this database hook implicitly enables
+                        read_special (see above) to support dump and restore
+                        streaming.
                     example: /var/lib/sqlite/users.db
                 restore_path:
                     type: string
@@ -1429,9 +1423,8 @@ properties:
                     description: |
                         Database name (required if using this hook). Or "all" to
                         dump all databases on the host. Note that using this
-                        database hook implicitly enables both read_special and
-                        one_file_system (see above) to support dump and restore
-                        streaming.
+                        database hook implicitly enables read_special (see
+                        above) to support dump and restore streaming.
                     example: users
                 hostname:
                     type: string
@@ -2266,7 +2259,45 @@ properties:
             can send the logs to a self-hosted instance or create an account at
             https://grafana.com/auth/sign-up/create-user. See borgmatic
             monitoring documentation for details.
-
+    sentry:
+        type: object
+        required: ['data_source_name_url', 'monitor_slug']
+        additionalProperties: false
+        properties:
+            data_source_name_url:
+                type: string
+                description: |
+                    Sentry Data Source Name (DSN) URL, associated with a
+                    particular Sentry project. Used to construct a cron URL,
+                    notified when a backup begins, ends, or errors.
+                example: https://5f80ec@o294220.ingest.us.sentry.io/203069
+            monitor_slug:
+                type: string
+                description: |
+                    Sentry monitor slug, associated with a particular Sentry
+                    project monitor. Used along with the data source name URL to
+                    construct a cron URL.
+                example: mymonitor
+            states:
+                type: array
+                items:
+                    type: string
+                    enum:
+                        - start
+                        - finish
+                        - fail
+                    uniqueItems: true
+                description: |
+                    List of one or more monitoring states to ping for: "start",
+                    "finish", and/or "fail". Defaults to pinging for all states.
+                example:
+                    - start
+                    - finish
+        description: |
+            Configuration for a monitoring integration with Sentry. You can use
+            a self-hosted instance via https://develop.sentry.dev/self-hosted/
+            or create a cloud-hosted account at https://sentry.io. See borgmatic
+            monitoring documentation for details.
     zfs:
         type: ["object", "null"]
         additionalProperties: false

borgmatic/hooks/data_source/bootstrap.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 
 logger = logging.getLogger(__name__)
@@ -22,19 +23,19 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
     Given a bootstrap configuration dict, a configuration dict, a log prefix, the borgmatic
-    configuration file paths, the borgmatic runtime directory, the configured source directories,
-    and whether this is a dry run, create a borgmatic manifest file to store the paths of the
-    configuration files used to create the archive. But skip this if the bootstrap
-    store_config_files option is False or if this is a dry run.
+    configuration file paths, the borgmatic runtime directory, the configured patterns, and whether
+    this is a dry run, create a borgmatic manifest file to store the paths of the configuration
+    files used to create the archive. But skip this if the bootstrap store_config_files option is
+    False or if this is a dry run.
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.
     '''
-    if hook_config.get('store_config_files') is False:
+    if hook_config and hook_config.get('store_config_files') is False:
         return []
 
     borgmatic_manifest_path = os.path.join(
@@ -55,8 +56,10 @@ def dump_data_sources(
             manifest_file,
         )
 
-    source_directories.extend(config_paths)
-    source_directories.append(os.path.join(borgmatic_runtime_directory, 'bootstrap'))
+    patterns.extend(borgmatic.borg.pattern.Pattern(config_path) for config_path in config_paths)
+    patterns.append(
+        borgmatic.borg.pattern.Pattern(os.path.join(borgmatic_runtime_directory, 'bootstrap'))
+    )
 
     return []
 

borgmatic/hooks/data_source/btrfs.py

@@ -1,10 +1,12 @@
 import collections
 import glob
+import json
 import logging
 import os
 import shutil
 import subprocess
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.execute
 import borgmatic.hooks.data_source.snapshot
@@ -26,13 +28,21 @@ def get_filesystem_mount_points(findmnt_command):
     findmnt_output = borgmatic.execute.execute_command_and_capture_output(
         tuple(findmnt_command.split(' '))
         + (
-            '-n',  # No headings.
             '-t',  # Filesystem type.
             'btrfs',
+            '--json',
+            '--list',  # Request a flat list instead of a nested subvolume hierarchy.
         )
     )
 
-    return tuple(line.rstrip().split(' ')[0] for line in findmnt_output.splitlines())
+    try:
+        return tuple(
+            filesystem['target'] for filesystem in json.loads(findmnt_output)['filesystems']
+        )
+    except json.JSONDecodeError as error:
+        raise ValueError(f'Invalid {findmnt_command} JSON output: {error}')
+    except KeyError as error:
+        raise ValueError(f'Invalid {findmnt_command} output: Missing key "{error}"')
 
 
 def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point):
@@ -63,41 +73,39 @@ def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point):
     )
 
 
-Subvolume = collections.namedtuple(
-    'Subvolume', ('path', 'contained_source_directories'), defaults=((),)
-)
+Subvolume = collections.namedtuple('Subvolume', ('path', 'contained_patterns'), defaults=((),))
 
 
-def get_subvolumes(btrfs_command, findmnt_command, source_directories=None):
+def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
     '''
-    Given a Btrfs command to run and a sequence of configured source directories, find the
-    intersection between the current Btrfs filesystem and subvolume mount points and the configured
-    borgmatic source directories. The idea is that these are the requested subvolumes to snapshot.
+    Given a Btrfs command to run and a sequence of configured patterns, find the intersection
+    between the current Btrfs filesystem and subvolume mount points and the paths of any patterns.
+    The idea is that these pattern paths represent the requested subvolumes to snapshot.
 
-    If the source directories is None, then return all subvolumes, sorted by path.
+    If patterns is None, then return all subvolumes, sorted by path.
 
     Return the result as a sequence of matching subvolume mount points.
     '''
-    candidate_source_directories = set(source_directories or ())
+    candidate_patterns = set(patterns or ())
     subvolumes = []
 
-    # For each filesystem mount point, find its subvolumes and match them again the given source
-    # directories to find the subvolumes to backup. And within this loop, sort the subvolumes from
-    # longest to shortest mount points, so longer mount points get a whack at the candidate source
-    # directory piñata before their parents do. (Source directories are consumed during this
-    # process, so no two datasets get the same contained source directories.)
+    # For each filesystem mount point, find its subvolumes and match them against the given patterns
+    # to find the subvolumes to backup. And within this loop, sort the subvolumes from longest to
+    # shortest mount points, so longer mount points get a whack at the candidate pattern piñata
+    # before their parents do. (Patterns are consumed during this process, so no two subvolumes end
+    # up with the same contained patterns.)
     for mount_point in get_filesystem_mount_points(findmnt_command):
         subvolumes.extend(
-            Subvolume(subvolume_path, contained_source_directories)
+            Subvolume(subvolume_path, contained_patterns)
             for subvolume_path in reversed(
                 get_subvolumes_for_filesystem(btrfs_command, mount_point)
             )
-            for contained_source_directories in (
-                borgmatic.hooks.data_source.snapshot.get_contained_directories(
-                    subvolume_path, candidate_source_directories
+            for contained_patterns in (
+                borgmatic.hooks.data_source.snapshot.get_contained_patterns(
+                    subvolume_path, candidate_patterns
                 ),
             )
-            if source_directories is None or contained_source_directories
+            if patterns is None or contained_patterns
         )
 
     return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path))
@@ -106,26 +114,24 @@ def get_subvolumes(btrfs_command, findmnt_command, source_directories=None):
 BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot-'
 
 
-def make_snapshot_path(subvolume_path):  # pragma: no cover
+def make_snapshot_path(subvolume_path):
     '''
     Given the path to a subvolume, make a corresponding snapshot path for it.
     '''
     return os.path.join(
         subvolume_path,
         f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
-        # Included so that the snapshot ends up in the Borg archive at the "original" subvolume
-        # path.
-        subvolume_path.lstrip(os.path.sep),
-    )
+        # Included so that the snapshot ends up in the Borg archive at the "original" subvolume path.
+    ) + subvolume_path.rstrip(os.path.sep)
 
 
-def make_snapshot_exclude_path(subvolume_path):  # pragma: no cover
+def make_snapshot_exclude_pattern(subvolume_path):  # pragma: no cover
     '''
-    Given the path to a subvolume, make a corresponding exclude path for its embedded snapshot path.
-    This is to work around a quirk of Btrfs: If you make a snapshot path as a child directory of a
-    subvolume, then the snapshot's own initial directory component shows up as an empty directory
-    within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and make a
-    snapshot of it at:
+    Given the path to a subvolume, make a corresponding exclude pattern for its embedded snapshot
+    path. This is to work around a quirk of Btrfs: If you make a snapshot path as a child directory
+    of a subvolume, then the snapshot's own initial directory component shows up as an empty
+    directory within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and
+    make a snapshot of it at:
 
         /mnt/.borgmatic-snapshot-1234/mnt
 
@@ -133,30 +139,52 @@ def make_snapshot_exclude_path(subvolume_path):  # pragma: no cover
 
         /mnt/.borgmatic-snapshot-1234/mnt/.borgmatic-snapshot-1234
 
-    So to prevent that from ending up in the Borg archive, this function produces its path for
-    exclusion.
+    So to prevent that from ending up in the Borg archive, this function produces an exclude pattern
+    to exclude that path.
     '''
     snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
 
-    return os.path.join(
-        subvolume_path,
-        snapshot_directory,
-        subvolume_path.lstrip(os.path.sep),
-        snapshot_directory,
+    return borgmatic.borg.pattern.Pattern(
+        os.path.join(
+            subvolume_path,
+            snapshot_directory,
+            subvolume_path.lstrip(os.path.sep),
+            snapshot_directory,
+        ),
+        borgmatic.borg.pattern.Pattern_type.EXCLUDE,
+        borgmatic.borg.pattern.Pattern_style.FNMATCH,
     )
 
 
-def make_borg_source_directory_path(subvolume_path, source_directory):  # pragma: no cover
+def make_borg_snapshot_pattern(subvolume_path, pattern):
     '''
-    Given the path to a subvolume and a source directory inside it, make a corresponding path for
-    the source directory within a snapshot path intended for giving to Borg.
+    Given the path to a subvolume and a pattern as a borgmatic.borg.pattern.Pattern instance whose
+    path is inside the subvolume, return a new Pattern with its path rewritten to be in a snapshot
+    path intended for giving to Borg.
+
+    Move any initial caret in a regular expression pattern path to the beginning, so as not to break
+    the regular expression.
     '''
-    return os.path.join(
+    initial_caret = (
+        '^'
+        if pattern.style == borgmatic.borg.pattern.Pattern_style.REGULAR_EXPRESSION
+        and pattern.path.startswith('^')
+        else ''
+    )
+
+    rewritten_path = initial_caret + os.path.join(
         subvolume_path,
         f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
         '.',  # Borg 1.4+ "slashdot" hack.
         # Included so that the source directory ends up in the Borg archive at its "original" path.
-        source_directory.lstrip(os.path.sep),
+        pattern.path.lstrip('^').lstrip(os.path.sep),
+    )
+
+    return borgmatic.borg.pattern.Pattern(
+        rewritten_path,
+        pattern.type,
+        pattern.style,
+        pattern.device,
     )
 
 
@@ -172,7 +200,7 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma:
         + (
             'subvolume',
             'snapshot',
-            '-r',  # Read-only,
+            '-r',  # Read-only.
             subvolume_path,
             snapshot_path,
         ),
@@ -186,16 +214,16 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
     Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic
-    configuration file paths, the borgmatic runtime directory, the configured source directories,
-    and whether this is a dry run, auto-detect and snapshot any Btrfs subvolume mount points listed
-    in the given source directories. Also update those source directories, replacing subvolume mount
-    points with corresponding snapshot directories so they get stored in the Borg archive instead.
-    Use the log prefix in any log entries.
+    configuration file paths, the borgmatic runtime directory, the configured patterns, and whether
+    this is a dry run, auto-detect and snapshot any Btrfs subvolume mount points listed in the given
+    patterns. Also update those patterns, replacing subvolume mount points with corresponding
+    snapshot directories so they get stored in the Borg archive instead. Use the log prefix in any
+    log entries.
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.
 
@@ -204,15 +232,15 @@ def dump_data_sources(
     dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
     logger.info(f'{log_prefix}: Snapshotting Btrfs subvolumes{dry_run_label}')
 
-    # Based on the configured source directories, determine Btrfs subvolumes to backup.
+    # Based on the configured patterns, determine Btrfs subvolumes to backup.
     btrfs_command = hook_config.get('btrfs_command', 'btrfs')
     findmnt_command = hook_config.get('findmnt_command', 'findmnt')
-    subvolumes = get_subvolumes(btrfs_command, findmnt_command, source_directories)
+    subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns)
 
     if not subvolumes:
         logger.warning(f'{log_prefix}: No Btrfs subvolumes found to snapshot{dry_run_label}')
 
-    # Snapshot each subvolume, rewriting source directories to use their snapshot paths.
+    # Snapshot each subvolume, rewriting patterns to use their snapshot paths.
     for subvolume in subvolumes:
         logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume.path} subvolume')
 
@@ -223,17 +251,16 @@ def dump_data_sources(
 
         snapshot_subvolume(btrfs_command, subvolume.path, snapshot_path)
 
-        for source_directory in subvolume.contained_source_directories:
+        for pattern in subvolume.contained_patterns:
+            snapshot_pattern = make_borg_snapshot_pattern(subvolume.path, pattern)
+
+            # Attempt to update the pattern in place, since pattern order matters to Borg.
             try:
-                source_directories.remove(source_directory)
+                patterns[patterns.index(pattern)] = snapshot_pattern
             except ValueError:
-                pass
+                patterns.append(snapshot_pattern)
 
-            source_directories.append(
-                make_borg_source_directory_path(subvolume.path, source_directory)
-            )
-
-        config.setdefault('exclude_patterns', []).append(make_snapshot_exclude_path(subvolume.path))
+        patterns.append(make_snapshot_exclude_pattern(subvolume.path))
 
     return []
 
@@ -257,8 +284,12 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_
     '''
     Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic runtime
     directory, and whether this is a dry run, delete any Btrfs snapshots created by borgmatic. Use
-    the log prefix in any log entries. If this is a dry run, then don't actually remove anything.
+    the log prefix in any log entries. If this is a dry run or Btrfs isn't configured in borgmatic's
+    configuration, then don't actually remove anything.
     '''
+    if hook_config is None:
+        return
+
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     btrfs_command = hook_config.get('btrfs_command', 'btrfs')

borgmatic/hooks/data_source/dump.py

@@ -16,17 +16,19 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam
     return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
 
 
-def make_data_source_dump_filename(dump_path, name, hostname=None):
+def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
     '''
-    Based on the given dump directory path, data source name, and hostname, return a filename to use
-    for the data source dump. The hostname defaults to localhost.
+    Based on the given dump directory path, data source name, hostname, and port, return a filename
+    to use for the data source dump. The hostname defaults to localhost.
 
     Raise ValueError if the data source name is invalid.
     '''
     if os.path.sep in name:
         raise ValueError(f'Invalid data source name {name}')
 
-    return os.path.join(dump_path, hostname or 'localhost', name)
+    return os.path.join(
+        dump_path, (hostname or 'localhost') + ('' if port is None else f':{port}'), name
+    )
 
 
 def create_parent_directory_for_dump(dump_path):

borgmatic/hooks/data_source/lvm.py

@@ -6,6 +6,7 @@ import os
 import shutil
 import subprocess
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.execute
 import borgmatic.hooks.data_source.snapshot
@@ -22,18 +23,17 @@ def use_streaming(hook_config, config, log_prefix):  # pragma: no cover
 
 BORGMATIC_SNAPSHOT_PREFIX = 'borgmatic-'
 Logical_volume = collections.namedtuple(
-    'Logical_volume', ('name', 'device_path', 'mount_point', 'contained_source_directories')
+    'Logical_volume', ('name', 'device_path', 'mount_point', 'contained_patterns')
 )
 
 
-def get_logical_volumes(lsblk_command, source_directories=None):
+def get_logical_volumes(lsblk_command, patterns=None):
     '''
-    Given an lsblk command to run and a sequence of configured source directories, find the
-    intersection between the current LVM logical volume mount points and the configured borgmatic
-    source directories. The idea is that these are the requested logical volumes to snapshot.
+    Given an lsblk command to run and a sequence of configured patterns, find the intersection
+    between the current LVM logical volume mount points and the paths of any patterns. The idea is
+    that these pattern paths represent the requested logical volumes to snapshot.
 
-    If source directories is None, include all logical volume mounts points, not just those in
-    source directories.
+    If patterns is None, include all logical volume mounts points, not just those in patterns.
 
     Return the result as a sequence of Logical_volume instances.
     '''
@@ -53,21 +53,26 @@ def get_logical_volumes(lsblk_command, source_directories=None):
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {lsblk_command} JSON output: {error}')
 
-    candidate_source_directories = set(source_directories or ())
+    candidate_patterns = set(patterns or ())
 
     try:
+        # Sort from longest to shortest mount points, so longer mount points get a whack at the
+        # candidate pattern piñata before their parents do. (Patterns are consumed below, so no two
+        # logical volumes end up with the same contained patterns.)
         return tuple(
-            Logical_volume(
-                device['name'], device['path'], device['mountpoint'], contained_source_directories
+            Logical_volume(device['name'], device['path'], device['mountpoint'], contained_patterns)
+            for device in sorted(
+                devices_info['blockdevices'],
+                key=lambda device: device['mountpoint'] or '',
+                reverse=True,
             )
-            for device in devices_info['blockdevices']
             if device['mountpoint'] and device['type'] == 'lvm'
-            for contained_source_directories in (
-                borgmatic.hooks.data_source.snapshot.get_contained_directories(
-                    device['mountpoint'], candidate_source_directories
+            for contained_patterns in (
+                borgmatic.hooks.data_source.snapshot.get_contained_patterns(
+                    device['mountpoint'], candidate_patterns
                 ),
             )
-            if not source_directories or contained_source_directories
+            if not patterns or contained_patterns
         )
     except KeyError as error:
         raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"')
@@ -89,6 +94,8 @@ def snapshot_logical_volume(
             '--snapshot',
             ('--extents' if '%' in snapshot_size else '--size'),
             snapshot_size,
+            '--permission',
+            'r',  # Read-only.
             '--name',
             snapshot_name,
             logical_volume_device,
@@ -117,6 +124,37 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # prag
     )
 
 
+def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
+    '''
+    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
+    path rewritten to be in a snapshot directory based on the given runtime directory.
+
+    Move any initial caret in a regular expression pattern path to the beginning, so as not to break
+    the regular expression.
+    '''
+    initial_caret = (
+        '^'
+        if pattern.style == borgmatic.borg.pattern.Pattern_style.REGULAR_EXPRESSION
+        and pattern.path.startswith('^')
+        else ''
+    )
+
+    rewritten_path = initial_caret + os.path.join(
+        normalized_runtime_directory,
+        'lvm_snapshots',
+        '.',  # Borg 1.4+ "slashdot" hack.
+        # Included so that the source directory ends up in the Borg archive at its "original" path.
+        pattern.path.lstrip('^').lstrip(os.path.sep),
+    )
+
+    return borgmatic.borg.pattern.Pattern(
+        rewritten_path,
+        pattern.type,
+        pattern.style,
+        pattern.device,
+    )
+
+
 DEFAULT_SNAPSHOT_SIZE = '10%ORIGIN'
 
 
@@ -126,16 +164,16 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
     Given an LVM configuration dict, a configuration dict, a log prefix, the borgmatic configuration
-    file paths, the borgmatic runtime directory, the configured source directories, and whether this
-    is a dry run, auto-detect and snapshot any LVM logical volume mount points listed in the given
-    source directories. Also update those source directories, replacing logical volume mount points
-    with corresponding snapshot directories so they get stored in the Borg archive instead. Use the
-    log prefix in any log entries.
+    file paths, the borgmatic runtime directory, the configured patterns, and whether this is a dry
+    run, auto-detect and snapshot any LVM logical volume mount points listed in the given patterns.
+    Also update those patterns, replacing logical volume mount points with corresponding snapshot
+    directories so they get stored in the Borg archive instead. Use the log prefix in any log
+    entries.
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.
 
@@ -146,7 +184,7 @@ def dump_data_sources(
 
     # List logical volumes to get their mount points.
     lsblk_command = hook_config.get('lsblk_command', 'lsblk')
-    requested_logical_volumes = get_logical_volumes(lsblk_command, source_directories)
+    requested_logical_volumes = get_logical_volumes(lsblk_command, patterns)
 
     # Snapshot each logical volume, rewriting source directories to use the snapshot paths.
     snapshot_suffix = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
@@ -196,22 +234,14 @@ def dump_data_sources(
             hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path
         )
 
-        # Update the path for each contained source directory, so Borg sees it within the
-        # mounted snapshot.
-        for source_directory in logical_volume.contained_source_directories:
-            try:
-                source_directories.remove(source_directory)
-            except ValueError:
-                pass
+        for pattern in logical_volume.contained_patterns:
+            snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
 
-            source_directories.append(
-                os.path.join(
-                    normalized_runtime_directory,
-                    'lvm_snapshots',
-                    '.',  # Borg 1.4+ "slashdot" hack.
-                    source_directory.lstrip(os.path.sep),
-                )
-            )
+            # Attempt to update the pattern in place, since pattern order matters to Borg.
+            try:
+                patterns[patterns.index(pattern)] = snapshot_pattern
+            except ValueError:
+                patterns.append(snapshot_pattern)
 
     return []
 
@@ -286,9 +316,12 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_
     '''
     Given an LVM configuration dict, a configuration dict, a log prefix, the borgmatic runtime
     directory, and whether this is a dry run, unmount and delete any LVM snapshots created by
-    borgmatic. Use the log prefix in any log entries. If this is a dry run, then don't actually
-    remove anything.
+    borgmatic. Use the log prefix in any log entries. If this is a dry run or LVM isn't configured
+    in borgmatic's configuration, then don't actually remove anything.
     '''
+    if hook_config is None:
+        return
+
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     # Unmount snapshots.

borgmatic/hooks/data_source/mariadb.py

@@ -3,6 +3,7 @@ import logging
 import os
 import shlex
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 from borgmatic.execute import (
     execute_command,
@@ -73,7 +74,10 @@ def execute_dump_command(
     '''
     database_name = database['name']
     dump_filename = dump.make_data_source_dump_filename(
-        dump_path, database['name'], database.get('hostname')
+        dump_path,
+        database['name'],
+        database.get('hostname'),
+        database.get('port'),
     )
 
     if os.path.exists(dump_filename):
@@ -114,6 +118,10 @@ def execute_dump_command(
     )
 
 
+def get_default_port(databases, config, log_prefix):  # pragma: no cover
+    return 3306
+
+
 def use_streaming(databases, config, log_prefix):
     '''
     Given a sequence of MariaDB database configuration dicts, a configuration dict (ignored), and a
@@ -128,7 +136,7 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
@@ -139,7 +147,8 @@ def dump_data_sources(
 
     Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
     pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
-    Also append the given source directories with the parent directory of the database dumps.
+    Also append the the parent directory of the database dumps to the given patterns list, so the
+    dumps actually get backed up.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
@@ -188,7 +197,11 @@ def dump_data_sources(
             )
 
     if not dry_run:
-        source_directories.append(os.path.join(borgmatic_runtime_directory, 'mariadb_databases'))
+        patterns.append(
+            borgmatic.borg.pattern.Pattern(
+                os.path.join(borgmatic_runtime_directory, 'mariadb_databases')
+            )
+        )
 
     return [process for process in processes if process]
 

borgmatic/hooks/data_source/mongodb.py

@@ -2,6 +2,7 @@ import logging
 import os
 import shlex
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks.data_source import dump
@@ -16,6 +17,10 @@ def make_dump_path(base_directory):  # pragma: no cover
     return dump.make_data_source_dump_path(base_directory, 'mongodb_databases')
 
 
+def get_default_port(databases, config, log_prefix):  # pragma: no cover
+    return 27017
+
+
 def use_streaming(databases, config, log_prefix):
     '''
     Given a sequence of MongoDB database configuration dicts, a configuration dict (ignored), and a
@@ -30,7 +35,7 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
@@ -41,7 +46,8 @@ def dump_data_sources(
 
     Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
     pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
-    Also append the given source directories with the parent directory of the database dumps.
+    Also append the the parent directory of the database dumps to the given patterns list, so the
+    dumps actually get backed up.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
 
@@ -51,7 +57,10 @@ def dump_data_sources(
     for database in databases:
         name = database['name']
         dump_filename = dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, database.get('hostname')
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            database.get('hostname'),
+            database.get('port'),
         )
         dump_format = database.get('format', 'archive')
 
@@ -71,7 +80,11 @@ def dump_data_sources(
             processes.append(execute_command(command, shell=True, run_to_completion=False))
 
     if not dry_run:
-        source_directories.append(os.path.join(borgmatic_runtime_directory, 'mongodb_databases'))
+        patterns.append(
+            borgmatic.borg.pattern.Pattern(
+                os.path.join(borgmatic_runtime_directory, 'mongodb_databases')
+            )
+        )
 
     return processes
 

borgmatic/hooks/data_source/mysql.py

@@ -3,6 +3,7 @@ import logging
 import os
 import shlex
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 from borgmatic.execute import (
     execute_command,
@@ -73,7 +74,10 @@ def execute_dump_command(
     '''
     database_name = database['name']
     dump_filename = dump.make_data_source_dump_filename(
-        dump_path, database['name'], database.get('hostname')
+        dump_path,
+        database['name'],
+        database.get('hostname'),
+        database.get('port'),
     )
 
     if os.path.exists(dump_filename):
@@ -113,6 +117,10 @@ def execute_dump_command(
     )
 
 
+def get_default_port(databases, config, log_prefix):  # pragma: no cover
+    return 3306
+
+
 def use_streaming(databases, config, log_prefix):
     '''
     Given a sequence of MySQL database configuration dicts, a configuration dict (ignored), and a
@@ -127,7 +135,7 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
@@ -138,7 +146,8 @@ def dump_data_sources(
 
     Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
     pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
-    Also append the given source directories with the parent directory of the database dumps.
+    Also append the the parent directory of the database dumps to the given patterns list, so the
+    dumps actually get backed up.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
@@ -187,7 +196,11 @@ def dump_data_sources(
             )
 
     if not dry_run:
-        source_directories.append(os.path.join(borgmatic_runtime_directory, 'mysql_databases'))
+        patterns.append(
+            borgmatic.borg.pattern.Pattern(
+                os.path.join(borgmatic_runtime_directory, 'mysql_databases')
+            )
+        )
 
     return [process for process in processes if process]
 

borgmatic/hooks/data_source/postgresql.py

@@ -5,6 +5,7 @@ import os
 import pathlib
 import shlex
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 from borgmatic.execute import (
     execute_command,
@@ -96,6 +97,10 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
     )
 
 
+def get_default_port(databases, config, log_prefix):  # pragma: no cover
+    return 5432
+
+
 def use_streaming(databases, config, log_prefix):
     '''
     Given a sequence of PostgreSQL database configuration dicts, a configuration dict (ignored), and
@@ -110,7 +115,7 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
@@ -121,7 +126,8 @@ def dump_data_sources(
 
     Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
     pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
-    Also append the given source directories with the parent directory of the database dumps.
+    Also append the the parent directory of the database dumps to the given patterns list, so the
+    dumps actually get backed up.
 
     Raise ValueError if the databases to dump cannot be determined.
     '''
@@ -151,7 +157,10 @@ def dump_data_sources(
                 for part in shlex.split(database.get('pg_dump_command') or default_dump_command)
             )
             dump_filename = dump.make_data_source_dump_filename(
-                dump_path, database_name, database.get('hostname')
+                dump_path,
+                database_name,
+                database.get('hostname'),
+                database.get('port'),
             )
             if os.path.exists(dump_filename):
                 logger.warning(
@@ -213,7 +222,11 @@ def dump_data_sources(
                 )
 
     if not dry_run:
-        source_directories.append(os.path.join(borgmatic_runtime_directory, 'postgresql_databases'))
+        patterns.append(
+            borgmatic.borg.pattern.Pattern(
+                os.path.join(borgmatic_runtime_directory, 'postgresql_databases')
+            )
+        )
 
     return processes
 

borgmatic/hooks/data_source/snapshot.py

@@ -3,28 +3,37 @@ import pathlib
 IS_A_HOOK = False
 
 
-def get_contained_directories(parent_directory, candidate_contained_directories):
+def get_contained_patterns(parent_directory, candidate_patterns):
     '''
-    Given a parent directory and a set of candidate directories potentially inside it, get the
-    subset of contained directories for which the parent directory is actually the parent, a
-    grandparent, the very same directory, etc. The idea is if, say, /var/log and /var/lib are
-    candidate contained directories, but there's a parent directory (logical volume, dataset,
-    subvolume, etc.) at /var, then /var is what we want to snapshot.
+    Given a parent directory and a set of candidate patterns potentially inside it, get the subset
+    of contained patterns for which the parent directory is actually the parent, a grandparent, the
+    very same directory, etc. The idea is if, say, /var/log and /var/lib are candidate pattern
+    paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then
+    /var is what we want to snapshot.
 
-    Also mutate the given set of candidate contained directories to remove any actually contained
-    directories from it. That way, this function can be called multiple times, successively
-    processing candidate directories until none are left—and avoiding assigning any candidate
-    directory to more than one parent directory.
+    For this to work, a candidate pattern path can't have any globs or other non-literal characters
+    in the initial portion of the path that matches the parent directory. For instance, a parent
+    directory of /var would match a candidate pattern path of /var/log/*/data, but not a pattern
+    path like /v*/log/*/data.
+
+    The one exception is that if a regular expression pattern path starts with "^", that will get
+    stripped off for purposes of matching against a parent directory.
+
+    As part of this, also mutate the given set of candidate patterns to remove any actually
+    contained patterns from it. That way, this function can be called multiple times, successively
+    processing candidate patterns until none are left—and avoiding assigning any candidate pattern
+    to more than one parent directory.
     '''
-    if not candidate_contained_directories:
+    if not candidate_patterns:
         return ()
 
-    contained = tuple(
+    contained_patterns = tuple(
         candidate
-        for candidate in candidate_contained_directories
-        if pathlib.PurePath(parent_directory) == pathlib.PurePath(candidate)
-        or pathlib.PurePath(parent_directory) in pathlib.PurePath(candidate).parents
+        for candidate in candidate_patterns
+        for candidate_path in (pathlib.PurePath(candidate.path.lstrip('^')),)
+        if pathlib.PurePath(parent_directory) == candidate_path
+        or pathlib.PurePath(parent_directory) in candidate_path.parents
     )
-    candidate_contained_directories -= set(contained)
+    candidate_patterns -= set(contained_patterns)
 
-    return contained
+    return contained_patterns

borgmatic/hooks/data_source/sqlite.py

@@ -2,6 +2,7 @@ import logging
 import os
 import shlex
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks.data_source import dump
@@ -16,6 +17,10 @@ def make_dump_path(base_directory):  # pragma: no cover
     return dump.make_data_source_dump_path(base_directory, 'sqlite_databases')
 
 
+def get_default_port(databases, config, log_prefix):  # pragma: no cover
+    return None  # SQLite doesn't use a port.
+
+
 def use_streaming(databases, config, log_prefix):
     '''
     Given a sequence of SQLite database configuration dicts, a configuration dict (ignored), and a
@@ -30,7 +35,7 @@ def dump_data_sources(
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
@@ -40,7 +45,8 @@ def dump_data_sources(
 
     Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
     pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
-    Also append the given source directories with the parent directory of the database dumps.
+    Also append the the parent directory of the database dumps to the given patterns list, so the
+    dumps actually get backed up.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
@@ -83,7 +89,11 @@ def dump_data_sources(
         processes.append(execute_command(command, shell=True, run_to_completion=False))
 
     if not dry_run:
-        source_directories.append(os.path.join(borgmatic_runtime_directory, 'sqlite_databases'))
+        patterns.append(
+            borgmatic.borg.pattern.Pattern(
+                os.path.join(borgmatic_runtime_directory, 'sqlite_databases')
+            )
+        )
 
     return processes
 

borgmatic/hooks/data_source/zfs.py

@@ -5,6 +5,7 @@ import os
 import shutil
 import subprocess
 
+import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.execute
 import borgmatic.hooks.data_source.snapshot
@@ -25,18 +26,17 @@ BORGMATIC_USER_PROPERTY = 'org.torsion.borgmatic:backup'
 
 Dataset = collections.namedtuple(
     'Dataset',
-    ('name', 'mount_point', 'auto_backup', 'contained_source_directories'),
+    ('name', 'mount_point', 'auto_backup', 'contained_patterns'),
     defaults=(False, ()),
 )
 
 
-def get_datasets_to_backup(zfs_command, source_directories):
+def get_datasets_to_backup(zfs_command, patterns):
     '''
-    Given a ZFS command to run and a sequence of configured source directories, find the
-    intersection between the current ZFS dataset mount points and the configured borgmatic source
-    directories. The idea is that these are the requested datasets to snapshot. But also include any
-    datasets tagged with a borgmatic-specific user property, whether or not they appear in source
-    directories.
+    Given a ZFS command to run and a sequence of configured patterns, find the intersection between
+    the current ZFS dataset mount points and the paths of any patterns. The idea is that these
+    pattern paths represent the requested datasets to snapshot. But also include any datasets tagged
+    with a borgmatic-specific user property, whether or not they appear in the patterns.
 
     Return the result as a sequence of Dataset instances, sorted by mount point.
     '''
@@ -54,9 +54,8 @@ def get_datasets_to_backup(zfs_command, source_directories):
 
     try:
         # Sort from longest to shortest mount points, so longer mount points get a whack at the
-        # candidate source directory piñata before their parents do. (Source directories are
-        # consumed during the second loop below, so no two datasets get the same contained source
-        # directories.)
+        # candidate pattern piñata before their parents do. (Patterns are consumed during the second
+        # loop below, so no two datasets end up with the same contained patterns.)
         datasets = sorted(
             (
                 Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ())
@@ -69,7 +68,7 @@ def get_datasets_to_backup(zfs_command, source_directories):
     except ValueError:
         raise ValueError(f'Invalid {zfs_command} list output')
 
-    candidate_source_directories = set(source_directories)
+    candidate_patterns = set(patterns)
 
     return tuple(
         sorted(
@@ -78,19 +77,22 @@ def get_datasets_to_backup(zfs_command, source_directories):
                     dataset.name,
                     dataset.mount_point,
                     dataset.auto_backup,
-                    contained_source_directories,
+                    contained_patterns,
                 )
                 for dataset in datasets
-                for contained_source_directories in (
+                for contained_patterns in (
                     (
-                        (dataset.mount_point,)
-                        if dataset.auto_backup
-                        else borgmatic.hooks.data_source.snapshot.get_contained_directories(
-                            dataset.mount_point, candidate_source_directories
+                        (
+                            (borgmatic.borg.pattern.Pattern(dataset.mount_point),)
+                            if dataset.auto_backup
+                            else ()
+                        )
+                        + borgmatic.hooks.data_source.snapshot.get_contained_patterns(
+                            dataset.mount_point, candidate_patterns
                         )
                     ),
                 )
-                if contained_source_directories
+                if contained_patterns
             ),
             key=lambda dataset: dataset.mount_point,
         )
@@ -153,22 +155,53 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
     )
 
 
+def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
+    '''
+    Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
+    path rewritten to be in a snapshot directory based on the given runtime directory.
+
+    Move any initial caret in a regular expression pattern path to the beginning, so as not to break
+    the regular expression.
+    '''
+    initial_caret = (
+        '^'
+        if pattern.style == borgmatic.borg.pattern.Pattern_style.REGULAR_EXPRESSION
+        and pattern.path.startswith('^')
+        else ''
+    )
+
+    rewritten_path = initial_caret + os.path.join(
+        normalized_runtime_directory,
+        'zfs_snapshots',
+        '.',  # Borg 1.4+ "slashdot" hack.
+        # Included so that the source directory ends up in the Borg archive at its "original" path.
+        pattern.path.lstrip('^').lstrip(os.path.sep),
+    )
+
+    return borgmatic.borg.pattern.Pattern(
+        rewritten_path,
+        pattern.type,
+        pattern.style,
+        pattern.device,
+    )
+
+
 def dump_data_sources(
     hook_config,
     config,
     log_prefix,
     config_paths,
     borgmatic_runtime_directory,
-    source_directories,
+    patterns,
     dry_run,
 ):
     '''
     Given a ZFS configuration dict, a configuration dict, a log prefix, the borgmatic configuration
-    file paths, the borgmatic runtime directory, the configured source directories, and whether this
-    is a dry run, auto-detect and snapshot any ZFS dataset mount points listed in the given source
-    directories and any dataset with a borgmatic-specific user property. Also update those source
-    directories, replacing dataset mount points with corresponding snapshot directories so they get
-    stored in the Borg archive instead. Use the log prefix in any log entries.
+    file paths, the borgmatic runtime directory, the configured patterns, and whether this is a dry
+    run, auto-detect and snapshot any ZFS dataset mount points listed in the given patterns and any
+    dataset with a borgmatic-specific user property. Also update those patterns, replacing dataset
+    mount points with corresponding snapshot directories so they get stored in the Borg archive
+    instead. Use the log prefix in any log entries.
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.
 
@@ -179,9 +212,9 @@ def dump_data_sources(
 
     # List ZFS datasets to get their mount points.
     zfs_command = hook_config.get('zfs_command', 'zfs')
-    requested_datasets = get_datasets_to_backup(zfs_command, source_directories)
+    requested_datasets = get_datasets_to_backup(zfs_command, patterns)
 
-    # Snapshot each dataset, rewriting source directories to use the snapshot paths.
+    # Snapshot each dataset, rewriting patterns to use the snapshot paths.
     snapshot_name = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
     normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory)
 
@@ -216,20 +249,14 @@ def dump_data_sources(
             hook_config.get('mount_command', 'mount'), full_snapshot_name, snapshot_mount_path
         )
 
-        for source_directory in dataset.contained_source_directories:
-            try:
-                source_directories.remove(source_directory)
-            except ValueError:
-                pass
+        for pattern in dataset.contained_patterns:
+            snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
 
-            source_directories.append(
-                os.path.join(
-                    normalized_runtime_directory,
-                    'zfs_snapshots',
-                    '.',  # Borg 1.4+ "slashdot" hack.
-                    source_directory.lstrip(os.path.sep),
-                )
-            )
+            # Attempt to update the pattern in place, since pattern order matters to Borg.
+            try:
+                patterns[patterns.index(pattern)] = snapshot_pattern
+            except ValueError:
+                patterns.append(snapshot_pattern)
 
     return []
 
@@ -283,9 +310,12 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_
     '''
     Given a ZFS configuration dict, a configuration dict, a log prefix, the borgmatic runtime
     directory, and whether this is a dry run, unmount and destroy any ZFS snapshots created by
-    borgmatic. Use the log prefix in any log entries. If this is a dry run, then don't actually
-    remove anything.
+    borgmatic. Use the log prefix in any log entries. If this is a dry run or ZFS isn't configured
+    in borgmatic's configuration, then don't actually remove anything.
     '''
+    if hook_config is None:
+        return
+
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     # Unmount snapshots.

borgmatic/hooks/dispatch.py

@@ -32,7 +32,11 @@ def call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs):
     Raise AttributeError if the function name is not found in the module.
     Raise anything else that the called function raises.
     '''
-    hook_config = config.get(hook_name) or config.get(f'{hook_name}_databases') or {}
+    if hook_name in config or f'{hook_name}_databases' in config:
+        hook_config = config.get(hook_name) or config.get(f'{hook_name}_databases') or {}
+    else:
+        hook_config = None
+
     module_name = hook_name.split('_databases')[0]
 
     # Probe for a data source or monitoring hook module corresponding to the hook name.

borgmatic/hooks/monitoring/ntfy.py

@@ -19,7 +19,6 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     Ping the configured Ntfy topic. Use the given configuration filename in any log entries.
     If this is a dry run, then don't actually ping anything.
     '''
-
     run_states = hook_config.get('states', ['fail'])
 
     if state.name.lower() in run_states:

borgmatic/hooks/monitoring/sentry.py

@@ -0,0 +1,79 @@
+import logging
+import re
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def initialize_monitor(
+    ping_url, config, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
+DATA_SOURCE_NAME_URL_PATTERN = re.compile(
+    '^(?P<protocol>.*)://(?P<username>.*)@(?P<hostname>.*)/(?P<project_id>.*)$'
+)
+
+
+def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    Construct and ping a Sentry cron URL, based on the configured DSN URL and monitor slug. Use the
+    given configuration filename in any log entries. If this is a dry run, then don't actually ping
+    anything.
+    '''
+    run_states = hook_config.get('states', ['start', 'finish', 'fail'])
+
+    if not state.name.lower() in run_states:
+        return
+
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+    data_source_name_url = hook_config.get('data_source_name_url')
+    monitor_slug = hook_config.get('monitor_slug')
+    match = DATA_SOURCE_NAME_URL_PATTERN.match(data_source_name_url)
+
+    if not match:
+        logger.warning(
+            'f{config_filename}: Invalid Sentry data source name URL: {data_source_name_url}'
+        )
+        return
+
+    cron_url = f'{match.group("protocol")}://{match.group("hostname")}/api/{match.group("project_id")}/cron/{monitor_slug}/{match.group("username")}/'
+
+    logger.info(f'{config_filename}: Pinging Sentry {state.name.lower()}{dry_run_label}')
+    logger.debug(f'{config_filename}: Using Sentry cron URL {cron_url}')
+
+    status = {
+        'start': 'in_progress',
+        'finish': 'ok',
+        'fail': 'error',
+    }.get(state.name.lower())
+
+    if not status:
+        logger.warning('f{config_filename}: Invalid Sentry state')
+        return
+
+    if dry_run:
+        return
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    try:
+        response = requests.post(f'{cron_url}?status={status}')
+        if not response.ok:
+            response.raise_for_status()
+    except requests.exceptions.RequestException as error:
+        logger.warning(f'{config_filename}: Sentry error: {error}')
+
+
+def destroy_monitor(
+    ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No destruction is necessary for this monitor.
+    '''
+    pass

borgmatic/logger.py

@@ -1,10 +1,9 @@
+import enum
 import logging
 import logging.handlers
 import os
 import sys
 
-import colorama
-
 
 def to_bool(arg):
     '''
@@ -33,7 +32,7 @@ def interactive_console():
 def should_do_markup(no_color, configs):
     '''
     Given the value of the command-line no-color argument, and a dict of configuration filename to
-    corresponding parsed configuration, determine if we should enable colorama marking up.
+    corresponding parsed configuration, determine if we should enable color marking up.
     '''
     if no_color:
         return False
@@ -93,30 +92,50 @@ class Console_no_color_formatter(logging.Formatter):
         return record.msg
 
 
+class Color(enum.Enum):
+    RESET = 0
+    RED = 31
+    GREEN = 32
+    YELLOW = 33
+    MAGENTA = 35
+    CYAN = 36
+
+
 class Console_color_formatter(logging.Formatter):
     def format(self, record):
         add_custom_log_levels()
 
-        color = {
-            logging.CRITICAL: colorama.Fore.RED,
-            logging.ERROR: colorama.Fore.RED,
-            logging.WARN: colorama.Fore.YELLOW,
-            logging.ANSWER: colorama.Fore.MAGENTA,
-            logging.INFO: colorama.Fore.GREEN,
-            logging.DEBUG: colorama.Fore.CYAN,
-        }.get(record.levelno)
+        color = (
+            {
+                logging.CRITICAL: Color.RED,
+                logging.ERROR: Color.RED,
+                logging.WARN: Color.YELLOW,
+                logging.ANSWER: Color.MAGENTA,
+                logging.INFO: Color.GREEN,
+                logging.DEBUG: Color.CYAN,
+            }
+            .get(record.levelno)
+            .value
+        )
 
         return color_text(color, record.msg)
 
 
+def ansi_escape_code(color):  # pragma: no cover
+    '''
+    Given a color value, produce the corresponding ANSI escape code.
+    '''
+    return f'\x1b[{color}m'
+
+
 def color_text(color, message):
     '''
-    Give colored text.
+    Given a color value and a message, return the message colored with ANSI escape codes.
     '''
     if not color:
         return message
 
-    return f'{color}{message}{colorama.Style.RESET_ALL}'
+    return f'{ansi_escape_code(color)}{message}{ansi_escape_code(Color.RESET.value)}'
 
 
 def add_logging_level(level_name, level_number):

docs/how-to/backup-your-databases.md

@@ -391,7 +391,7 @@ with the repository's path or its label as configured in your borgmatic
 configuration file.
 
 ```bash
-borgmatic restore --repository repo.borg --archive host-2023-...
+borgmatic restore --repository repo.borg --archive latest
 ```
 
 ### Restore particular databases
@@ -401,7 +401,7 @@ restore one of them, use the `--database` flag to select one or more
 databases. For instance:
 
 ```bash
-borgmatic restore --archive host-2023-... --database users --database orders
+borgmatic restore --archive latest --database users --database orders
 ```
 
 <span class="minilink minilink-addedin">New in version 1.7.6</span> You can
@@ -409,29 +409,99 @@ also restore individual databases even if you dumped them as "all"—as long as
 you dumped them into separate files via use of the "format" option. See above
 for more information.
 
+### Restore databases sharing a name
+
+<span class="minilink minilink-addedin">New in version 1.9.5</span> If you've
+backed up multiple databases that happen to share the same name but different
+hostnames, ports, or hooks, you can include additional flags to disambiguate
+which database you'd like to restore. For instance, let's say you've backed up
+the following configured databases:
+
+```yaml
+postgresql_databases:
+    - name: users
+      hostname: host1.example.org
+    - name: users
+      hostname: host2.example.org
+```
+
+... then you can run the following command to restore only one of them:
+
+```bash
+borgmatic restore --archive latest --database users --original-hostname host1.example.org
+```
+
+This selects a `users` database to restore, but only if it originally came
+from the host `host1.example.org`. This command won't restore `users`
+databases from any other hosts.
+
+Here's another example configuration:
+
+```yaml
+postgresql_databases:
+    - name: users
+      hostname: example.org
+      port: 5433
+    - name: users
+      hostname: example.org
+      port: 5434
+```
+
+And a command to restore just one of the databases:
+
+```bash
+borgmatic restore --archive latest --database users --original-port 5433
+```
+
+That restores a `users` database only if it originally came from port `5433`
+*and* if that port is in borgmatic's configuration, e.g. `port: 5433`.
+
+Finally, check out this configuration:
+
+```yaml
+postgresql_databases:
+    - name: users
+      hostname: example.org
+mariadb_databases:
+    - name: users
+      hostname: example.org
+```
+
+And to select just one of the databases to restore:
+
+```bash
+borgmatic restore --archive latest --database users --hook postgresql
+```
+
+That restores a `users` database only if it was dumped using the
+`postgresql_databases:` data source hook. This command won't restore `users`
+databases that were dumped using other hooks.
+
+Note that these flags don't change the hostname or port to which the database
+is actually restored. For that, see below about restoring to an alternate
+host.
+
 
 ### Restore all databases
 
 To restore all databases:
 
 ```bash
-borgmatic restore --archive host-2023-... --database all
+borgmatic restore --archive latest --database all
 ```
 
 Or omit the `--database` flag entirely:
 
 
 ```bash
-borgmatic restore --archive host-2023-...
+borgmatic restore --archive latest
 ```
 
-Prior to borgmatic version 1.7.6, this restores a combined "all" database
-dump from the archive.
-
 <span class="minilink minilink-addedin">New in version 1.7.6</span> Restoring
 "all" databases restores each database found in the selected archive. That
 includes any combined dump file named "all" and any other individual database
-dumps found in the archive.
+dumps found in the archive. Prior to borgmatic version 1.7.6, restoring "all"
+only restored a combined "all" database dump from the archive.
 
 
 ### Restore particular schemas
@@ -466,21 +536,35 @@ postgresql_databases:
         restore_password: trustsome1
 ```
 
+### Manual restoration
 
-### Limitations
+If you prefer to restore a database without the help of borgmatic, first
+[extract](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) an
+archive containing a database dump.
+
+borgmatic extracts the dump file into the `borgmatic/` directory within the
+extraction destination path. For example, if you're extracting to `/tmp`, then
+the dump will be in `/tmp/borgmatic/`.
+
+<span class="minilink minilink-addedin">Prior to version 1.9.0</span> borgmatic
+extracts the dump file into the *`username`*`/.borgmatic/` directory within the
+extraction destination path, where *`username`* is the user that created the
+backup. For example, if you created the backup with the `root` user and you're
+extracting to `/tmp`, then the dump will be in `/tmp/root/.borgmatic`.
+
+After extraction, you can manually restore the dump file using native database
+commands like `pg_restore`, `mysql`, `mongorestore`, `sqlite`, or similar.
+
+Also see the documentation on [listing database
+dumps](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#listing-database-dumps).
+
+
+## Limitations
 
 There are a few important limitations with borgmatic's current database
-restoration feature that you should know about:
+hooks that you should know about:
 
-1. You must restore as the same Unix user that created the archive containing
-the database dump. That's because the user's home directory path is encoded
-into the path of the database dump within the archive.
-2. As mentioned above, borgmatic can only restore a database that's defined in
-borgmatic's own configuration file. So include your configuration file in
-backups to avoid getting caught without a way to restore a database.
-3. borgmatic does not currently support backing up or restoring multiple
-databases that share the exact same name on different hosts.
-4. When database hooks are enabled, borgmatic instructs Borg to consume
+1. When database hooks are enabled, borgmatic instructs Borg to consume
 special files (via `--read-special`) to support database dump
 streaming—regardless of the value of your `read_special` configuration option.
 And because this can cause Borg to hang, borgmatic also automatically excludes
@@ -495,7 +579,10 @@ exclude them. <span class="minilink minilink-addedin">Prior to version
 1.7.3</span>Special files were not auto-excluded, and you were responsible for
 excluding them yourself. Common directories to exclude are `/dev` and `/run`,
 but that may not be exhaustive.
-5. <span class="minilink minilink-addedin">Prior to version 1.9.0</span>
+2. <span class="minilink minilink-addedin">Prior to version 1.9.5</span>
+borgmatic did not support backing up or restoring multiple databases that
+shared the exact same name on different hosts or with different ports.
+3. <span class="minilink minilink-addedin">Prior to version 1.9.0</span>
 Database hooks also implicitly enabled the `one_file_system` option, which
 meant Borg wouldn't cross filesystem boundaries when looking for files to
 backup. When borgmatic was running in a container, this often required a
@@ -503,25 +590,16 @@ work-around to explicitly add each mounted backup volume to
 `source_directories` instead of relying on Borg to include them implicitly via
 a parent directory. But as of borgmatic 1.9.0, `one_file_system` is no longer
 auto-enabled and such work-arounds aren't necessary.
-
-
-### Manual restoration
-
-If you prefer to restore a database without the help of borgmatic, first
-[extract](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) an
-archive containing a database dump.
-
-borgmatic extracts the dump file into the *`username`*`/.borgmatic/` directory
-within the extraction destination path, where *`username`* is the user that
-created the backup. For example, if you created the backup with the `root`
-user and you're extracting to `/tmp`, then the dump will be in
-`/tmp/root/.borgmatic`.
-
-After extraction, you can manually restore the dump file using native database
-commands like `pg_restore`, `mysql`, `mongorestore`, `sqlite`, or similar.
-
-Also see the documentation on [listing database
-dumps](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#listing-database-dumps).
+4. <span class="minilink minilink-addedin">Prior to version 1.9.0</span> You
+must restore as the same Unix user that created the archive containing the
+database dump. That's because the user's home directory path is encoded into
+the path of the database dump within the archive.
+5. <span class="minilink minilink-addedin">Prior to version 1.7.15</span> As
+mentioned above, borgmatic can only restore a database that's defined in
+borgmatic's own configuration file. So include your configuration files in
+backups to avoid getting caught without a way to restore a database. But
+starting from version 1.7.15, borgmatic includes your configuration files
+automatically.
 
 
 ## Preparation and cleanup hooks

docs/how-to/monitor-your-backups.md

@@ -47,6 +47,7 @@ them as backups happen:
  * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook)
  * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook)
  * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook)
+ * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook)
  * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook)
  * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook)
 
@@ -361,10 +362,49 @@ pushover:
 ```
 
 
+## Sentry hook
+
+<span class="minilink minilink-addedin">New in version 1.9.7</span>
+[Sentry](https://sentry.io/) is an application monitoring service that
+includes cron-style monitoring (either cloud-hosted or
+[self-hosted](https://develop.sentry.dev/self-hosted/)).
+
+To get started, create a [Sentry cron
+monitor](https://docs.sentry.io/product/crons/) in the Sentry UI. Under
+"Instrument your monitor," select "Sentry CLI" and copy the URL value for the
+displayed
+[`SENTRY_DSN`](https://docs.sentry.io/concepts/key-terms/dsn-explainer/)
+environment variable into borgmatic's Sentry `data_source_name_url`
+configuration option. For example:
+
+```
+sentry:
+    data_source_name_url: https://5f80ec@o294220.ingest.us.sentry.io/203069
+    monitor_slug: mymonitor
+```
+
+The `monitor_slug` value comes from the "Monitor Slug" under "Cron Details" on
+the same Sentry monitor page.
+
+With this configuration, borgmatic pings Sentry whenever borgmatic starts,
+finishes, or fails, but only when any of the `create`, `prune`, `compact`, or
+`check` actions are run. You can optionally override the start/finish/fail
+behavior with the `states` configuration option. For instance, to only ping
+Sentry on failure:
+
+```
+sentry:
+    data_source_name_url: https://5f80ec@o294220.ingest.us.sentry.io/203069
+    monitor_slug: mymonitor
+    states:
+      - fail
+```
+
+
 ## ntfy hook
 
 <span class="minilink minilink-addedin">New in version 1.6.3</span>
-[ntfy](https://ntfy.sh) is a free, simple, service (either hosted or
+[ntfy](https://ntfy.sh) is a free, simple, service (either cloud-hosted or
 self-hosted) which offers simple pub/sub push notifications to multiple
 platforms including [web](https://ntfy.sh/stats),
 [Android](https://play.google.com/store/apps/details?id=io.heckel.ntfy) and

docs/how-to/snapshot-your-filesystems.md

@@ -53,6 +53,9 @@ You have a couple of options for borgmatic to find and backup your ZFS datasets:
 
  * For any dataset you'd like backed up, add its mount point to borgmatic's
    `source_directories` option.
+ * <span class="minilink minilink-addedin">New in version 1.9.6</span> Or
+   include the mount point with borgmatic's `patterns` or `patterns_from`
+   options.
  * Or set the borgmatic-specific user property
    `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs
    set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find
@@ -63,7 +66,7 @@ like particular datasets to be backed up only for particular configuration
 files, use the `source_directories` option instead of the user property.
 
 During a backup, borgmatic automatically snapshots these discovered datasets
-(non-recursively), temporary mounts the snapshots within its [runtime
+(non-recursively), temporarily mounts the snapshots within its [runtime
 directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory),
 and includes the snapshotted files in the paths sent to Borg. borgmatic is also
 responsible for cleaning up (destroying) these snapshots after a backup
@@ -84,6 +87,14 @@ let's say you add `/var/log` and `/var/lib` to your source directories, but
 accordingly. This also works even with nested datasets; borgmatic selects
 the dataset that's the "closest" parent to your source directories.
 
+<span class="minilink minilink-addedin">New in version 1.9.6</span> When using
+[patterns](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns),
+the initial portion of a pattern's path that you intend borgmatic to match
+against a dataset can't have globs or other non-literal characters in it—or it
+won't actually match. For instance, a mount point of `/var` would match a
+pattern of `+ fm:/var/*/data`, but borgmatic isn't currently smart enough to
+match `/var` to a pattern like `+ fm:/v*/lib/data`.
+
 <span class="minilink minilink-addedin">With Borg version 1.2 and
 earlier</span>Snapshotted files are instead stored at a path dependent on the
 [runtime
@@ -133,10 +144,14 @@ feedback](https://torsion.org/borgmatic/#issues) you have on this feature.
 #### Subvolume discovery
 
 For any subvolume you'd like backed up, add its path to borgmatic's
-`source_directories` option. During a backup, borgmatic snapshots these
-subvolumes (non-recursively) and includes the snapshotted files in the paths
-sent to Borg. borgmatic is also responsible for cleaning up (deleting) these
-snapshots after a backup completes.
+`source_directories` option.
+
+<span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
+the mount point with borgmatic's `patterns` or `patterns_from` options.
+
+During a backup, borgmatic snapshots these subvolumes (non-recursively) and
+includes the snapshotted files in the paths sent to Borg. borgmatic is also
+responsible for cleaning up (deleting) these snapshots after a backup completes.
 
 borgmatic is smart enough to look at the parent (and grandparent, etc.)
 directories of each of your `source_directories` to discover any subvolumes.
@@ -146,6 +161,14 @@ snapshot `/var` accordingly. This also works even with nested subvolumes;
 borgmatic selects the subvolume that's the "closest" parent to your source
 directories.
 
+<span class="minilink minilink-addedin">New in version 1.9.6</span> When using
+[patterns](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns),
+the initial portion of a pattern's path that you intend borgmatic to match
+against a subvolume can't have globs or other non-literal characters in it—or it
+won't actually match. For instance, a subvolume of `/var` would match a pattern
+of `+ fm:/var/*/data`, but borgmatic isn't currently smart enough to match
+`/var` to a pattern like `+ fm:/v*/lib/data`.
+
 Additionally, borgmatic rewrites the snapshot file paths so that they appear
 at their original subvolume locations in a Borg archive. For instance, if your
 subvolume exists at `/var/subvolume`, then the snapshotted files will appear
@@ -215,12 +238,12 @@ too small (and LVM isn't configured to grow snapshots automatically), then the
 snapshots will fail to allocate enough space, resulting in a broken backup.
 
 If not specified, the `snapshot_size` option defaults to `10%ORIGIN`, which
-means 10% of the size of logical volume being snapshotted. See the [`lvcreate
---size` and `--extents`
+means 10% of the size of the logical volume being snapshotted. See the
+[`lvcreate --size` and `--extents`
 documentation](https://www.man7.org/linux/man-pages/man8/lvcreate.8.html) for
 more information about possible values here. (Under the hood, borgmatic uses
-`lvcreate --extents` if the `snapshot_size` is a percentage value, and
-`lvcreate --size` otherwise.)
+`lvcreate --extents` if the `snapshot_size` is a percentage value, and `lvcreate
+--size` otherwise.)
 
 
 #### Logical volume discovery
@@ -228,12 +251,14 @@ more information about possible values here. (Under the hood, borgmatic uses
 For any logical volume you'd like backed up, add its mount point to
 borgmatic's `source_directories` option.
 
-During a backup, borgmatic automatically snapshots these discovered logical
-volumes (non-recursively), temporary mounts the snapshots within its [runtime
-directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory),
-and includes the snapshotted files in the paths sent to Borg. borgmatic is
-also responsible for cleaning up (deleting) these snapshots after a backup
-completes.
+<span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
+the mount point with borgmatic's `patterns` or `patterns_from` options.
+
+During a backup, borgmatic automatically snapshots these discovered logical volumes
+(non-recursively), temporarily mounts the snapshots within its [runtime
+directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory), and
+includes the snapshotted files in the paths sent to Borg. borgmatic is also responsible for cleaning
+up (deleting) these snapshots after a backup completes.
 
 borgmatic is smart enough to look at the parent (and grandparent, etc.)
 directories of each of your `source_directories` to discover any logical
@@ -241,6 +266,14 @@ volumes. For instance, let's say you add `/var/log` and `/var/lib` to your
 source directories, but `/var` is a logical volume. borgmatic will discover
 that and snapshot `/var` accordingly.
 
+<span class="minilink minilink-addedin">New in version 1.9.6</span> When using
+[patterns](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns),
+the initial portion of a pattern's path that you intend borgmatic to match
+against a logical volume can't have globs or other non-literal characters in
+it—or it won't actually match. For instance, a logical volume of `/var` would
+match a pattern of `+ fm:/var/*/data`, but borgmatic isn't currently smart
+enough to match `/var` to a pattern like `+ fm:/v*/lib/data`.
+
 Additionally, borgmatic rewrites the snapshot file paths so that they appear
 at their original logical volume locations in a Borg archive. For instance, if
 your logical volume is mounted at `/var/lvolume`, then the snapshotted files

pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "borgmatic"
-version = "1.9.4"
+version = "1.9.7"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]
 description = "Simple, configuration-driven backup software for servers and workstations"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 classifiers=[
     "Development Status :: 5 - Production/Stable",
     "Environment :: Console",
@@ -17,11 +17,10 @@ classifiers=[
     "Topic :: System :: Archiving :: Backup",
 ]
 dependencies = [
-  "colorama>=0.4.1,<0.5",
-  "jsonschema",
-  "packaging",
-  "requests",
-  "ruamel.yaml>0.15.0",
+    "jsonschema",
+    "packaging",
+    "requests",
+    "ruamel.yaml>0.15.0",
 ]
 
 [project.scripts]

scripts/run-full-tests

@@ -17,12 +17,9 @@ if [ -z "$TEST_CONTAINER" ]; then
     exit 1
 fi
 
-apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
-    py3-ruamel.yaml py3-ruamel.yaml.clib py3-yaml bash sqlite fish
-# If certain dependencies of black are available in this version of Alpine, install them.
-apk add --no-cache py3-typed-ast py3-regex || true
-python3 -m pip install --no-cache --upgrade pip==24.2 setuptools==72.1.0
-pip3 install --ignore-installed tox==4.11.3
+apk add --no-cache python3 py3-pip borgbackup postgresql17-client mariadb-client mongodb-tools \
+    py3-mongo py3-regex py3-ruamel.yaml py3-ruamel.yaml.clib py3-tox py3-typed-ast py3-yaml bash \
+    sqlite fish
 export COVERAGE_FILE=/tmp/.coverage
 
 tox --workdir /tmp/.tox --sitepackages

test_requirements.txt

@@ -6,7 +6,6 @@ certifi==2024.7.4
 chardet==5.2.0
 click==8.1.7
 codespell==2.2.6
-colorama==0.4.6
 coverage==7.5.1
 flake8==7.0.0
 flake8-quotes==3.4.0

tests/end-to-end/commands/fake_findmnt.py

@@ -4,29 +4,38 @@ import sys
 
 def parse_arguments(*unparsed_arguments):
     parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument('-n', dest='headings', action='store_false', default=True)
     parser.add_argument('-t', dest='type')
+    parser.add_argument('--json', action='store_true')
+    parser.add_argument('--list', action='store_true')
 
     return parser.parse_args(unparsed_arguments)
 
 
-BUILTIN_FILESYSTEM_MOUNT_LINES = (
-    '/mnt/subvolume /dev/loop1 btrfs rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/',
-)
+BUILTIN_FILESYSTEM_MOUNT_OUTPUT = '''{
+       "filesystems": [
+          {
+             "target": "/mnt/subvolume",
+             "source": "/dev/loop0",
+             "fstype": "btrfs",
+             "options": "rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/"
+          }
+       ]
+    }
+    '''
 
 
-def print_filesystem_mounts(arguments):
-    for line in BUILTIN_FILESYSTEM_MOUNT_LINES:
-        print(line)
+def print_filesystem_mounts():
+    print(BUILTIN_FILESYSTEM_MOUNT_OUTPUT)
 
 
 def main():
     arguments = parse_arguments(*sys.argv[1:])
 
-    assert not arguments.headings
     assert arguments.type == 'btrfs'
+    assert arguments.json
+    assert arguments.list
 
-    print_filesystem_mounts(arguments)
+    print_filesystem_mounts()
 
 
 if __name__ == '__main__':

tests/end-to-end/commands/fake_lvcreate.py

@@ -9,6 +9,7 @@ def parse_arguments(*unparsed_arguments):
     parser.add_argument('--snapshot', action='store_true', required=True)
     parser.add_argument('--extents')
     parser.add_argument('--size')
+    parser.add_argument('--permission', required=True)
     parser.add_argument('--name', dest='snapshot_name', required=True)
     parser.add_argument('logical_volume_device')
 

tests/end-to-end/docker-compose.yaml

@@ -1,50 +1,50 @@
 services:
   postgresql:
-    image: docker.io/postgres:13.1-alpine
+    image: docker.io/postgres:17.2-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   postgresql2:
-    image: docker.io/postgres:13.1-alpine
+    image: docker.io/postgres:17.2-alpine
     environment:
       POSTGRES_PASSWORD: test2
       POSTGRES_DB: test
     command: docker-entrypoint.sh -p 5433
   mariadb:
-    image: docker.io/mariadb:10.11.4
+    image: docker.io/mariadb:11.4.4
     environment:
       MARIADB_ROOT_PASSWORD: test
       MARIADB_DATABASE: test
   mariadb2:
-    image: docker.io/mariadb:10.11.4
+    image: docker.io/mariadb:11.4.4
     environment:
       MARIADB_ROOT_PASSWORD: test2
       MARIADB_DATABASE: test
     command: docker-entrypoint.sh --port=3307
   not-actually-mysql:
-    image: docker.io/mariadb:10.11.4
+    image: docker.io/mariadb:11.4.4
     environment:
       MARIADB_ROOT_PASSWORD: test
       MARIADB_DATABASE: test
   not-actually-mysql2:
-    image: docker.io/mariadb:10.11.4
+    image: docker.io/mariadb:11.4.4
     environment:
       MARIADB_ROOT_PASSWORD: test2
       MARIADB_DATABASE: test
     command: docker-entrypoint.sh --port=3307
   mongodb:
-    image: docker.io/mongo:5.0.5
+    image: docker.io/mongo:7.0.16
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
   mongodb2:
-    image: docker.io/mongo:5.0.5
+    image: docker.io/mongo:7.0.16
     environment:
       MONGO_INITDB_ROOT_USERNAME: root2
       MONGO_INITDB_ROOT_PASSWORD: test2
     command: docker-entrypoint.sh --port=27018
   tests:
-    image: docker.io/alpine:3.13
+    image: docker.io/alpine:3.21
     environment:
       TEST_CONTAINER: true
     volumes:

tests/end-to-end/test_borgmatic.py

@@ -5,12 +5,14 @@ import subprocess
 import sys
 import tempfile
 
+import pytest
 
-def generate_configuration(config_path, repository_path):
+
+def generate_configuration_with_source_directories(config_path, repository_path):
     '''
     Generate borgmatic configuration into a file at the config path, and update the defaults so as
-    to work for testing (including injecting the given repository path and tacking on an encryption
-    passphrase).
+    to work for testing, including updating the source directories, injecting the given repository
+    path, and tacking on an encryption passphrase.
     '''
     subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
     config = (
@@ -19,17 +21,52 @@ def generate_configuration(config_path, repository_path):
         .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
         .replace('- path: /mnt/backup', '')
         .replace('label: local', '')
+        .replace('- /home/user/path with spaces', '')
         .replace('- /home', f'- {config_path}')
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')
-        + 'encryption_passphrase: "test"'
+        + '\nencryption_passphrase: "test"'
+        # Disable automatic storage of config files so we can test storage and extraction manually.
+        + '\nbootstrap:\n  store_config_files: false'
     )
     config_file = open(config_path, 'w')
     config_file.write(config)
     config_file.close()
 
 
-def test_borgmatic_command():
+def generate_configuration_with_patterns(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing, including adding patterns, injecting the given repository path, and tacking
+    on an encryption passphrase.
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('source_directories:', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', '')
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + f'\npatterns: ["R {config_path}"]'
+        + '\nencryption_passphrase: "test"'
+        # Disable automatic storage of config files so we can test storage and extraction manually.
+        + '\nbootstrap:\n  store_config_files: false'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+@pytest.mark.parametrize(
+    'generate_configuration',
+    (generate_configuration_with_source_directories, generate_configuration_with_patterns),
+)
+def test_borgmatic_command(generate_configuration):
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()
     repository_path = os.path.join(temporary_directory, 'test.borg')

tests/end-to-end/test_database.py

@@ -16,6 +16,8 @@ def write_configuration(
     repository_path,
     user_runtime_directory,
     postgresql_dump_format='custom',
+    postgresql_all_dump_format=None,
+    mariadb_mysql_all_dump_format=None,
     mongodb_dump_format='archive',
 ):
     '''
@@ -23,6 +25,13 @@ def write_configuration(
     for testing. This includes injecting the given repository path, borgmatic source directory for
     storing database dumps, dump format (for PostgreSQL), and encryption passphrase.
     '''
+    postgresql_all_format_option = (
+        f'format: {postgresql_all_dump_format}' if postgresql_all_dump_format else ''
+    )
+    mariadb_mysql_dump_format_option = (
+        f'format: {mariadb_mysql_all_dump_format}' if mariadb_mysql_all_dump_format else ''
+    )
+
     config_yaml = f'''
 source_directories:
     - {source_directory}
@@ -39,11 +48,7 @@ postgresql_databases:
       password: test
       format: {postgresql_dump_format}
     - name: all
-      hostname: postgresql
-      username: postgres
-      password: test
-    - name: all
-      format: custom
+      {postgresql_all_format_option}
       hostname: postgresql
       username: postgres
       password: test
@@ -53,11 +58,7 @@ mariadb_databases:
       username: root
       password: test
     - name: all
-      hostname: mariadb
-      username: root
-      password: test
-    - name: all
-      format: sql
+      {mariadb_mysql_dump_format_option}
       hostname: mariadb
       username: root
       password: test
@@ -67,11 +68,7 @@ mysql_databases:
       username: root
       password: test
     - name: all
-      hostname: not-actually-mysql
-      username: root
-      password: test
-    - name: all
-      format: sql
+      {mariadb_mysql_dump_format_option}
       hostname: not-actually-mysql
       username: root
       password: test
@@ -97,12 +94,21 @@ sqlite_databases:
     return ruamel.yaml.YAML(typ='safe').load(config_yaml)
 
 
+@pytest.mark.parametrize(
+    'postgresql_all_dump_format,mariadb_mysql_all_dump_format',
+    (
+        (None, None),
+        ('custom', 'sql'),
+    ),
+)
 def write_custom_restore_configuration(
     source_directory,
     config_path,
     repository_path,
     user_runtime_directory,
     postgresql_dump_format='custom',
+    postgresql_all_dump_format=None,
+    mariadb_mysql_all_dump_format=None,
     mongodb_dump_format='archive',
 ):
     '''

tests/unit/actions/test_check.py

@@ -2,6 +2,7 @@ import pytest
 from flexmock import flexmock
 
 from borgmatic.actions import check as module
+from borgmatic.borg.pattern import Pattern
 
 
 def test_parse_checks_returns_them_as_tuple():
@@ -557,14 +558,20 @@ def test_collect_spot_check_source_paths_parses_borg_output():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
-    flexmock(module.borgmatic.actions.create).should_receive(
-        'process_source_directories'
-    ).and_return(['foo', 'bar'])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+        [Pattern('foo'), Pattern('bar')]
+    )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        source_directories=['foo', 'bar'],
+        patterns=[Pattern('foo'), Pattern('bar')],
         local_borg_version=object,
         global_arguments=object,
         borgmatic_runtime_directory='/run/borgmatic',
@@ -572,7 +579,7 @@ def test_collect_spot_check_source_paths_parses_borg_output():
         remote_path=object,
         list_files=True,
         stream_processes=True,
-    ).and_return((('borg', 'create'), ('repo::archive',), flexmock(), flexmock()))
+    ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
     flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
         flexmock()
     )
@@ -599,14 +606,20 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': False}
     )
-    flexmock(module.borgmatic.actions.create).should_receive(
-        'process_source_directories'
-    ).and_return(['foo', 'bar'])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+        [Pattern('foo'), Pattern('bar')]
+    )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        source_directories=['foo', 'bar'],
+        patterns=[Pattern('foo'), Pattern('bar')],
         local_borg_version=object,
         global_arguments=object,
         borgmatic_runtime_directory='/run/borgmatic',
@@ -614,7 +627,7 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
         remote_path=object,
         list_files=True,
         stream_processes=False,
-    ).and_return((('borg', 'create'), ('repo::archive',), flexmock(), flexmock()))
+    ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
     flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
         flexmock()
     )
@@ -641,14 +654,20 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
-    flexmock(module.borgmatic.actions.create).should_receive(
-        'process_source_directories'
-    ).and_return(['foo', 'bar'])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+        [Pattern('foo'), Pattern('bar')]
+    )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        source_directories=['foo', 'bar'],
+        patterns=[Pattern('foo'), Pattern('bar')],
         local_borg_version=object,
         global_arguments=object,
         borgmatic_runtime_directory='/run/borgmatic',
@@ -656,7 +675,7 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
         remote_path=object,
         list_files=True,
         stream_processes=True,
-    ).and_return((('borg', 'create'), ('repo::archive',), flexmock(), flexmock()))
+    ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
     flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
         flexmock()
     )
@@ -683,14 +702,20 @@ def test_collect_spot_check_source_paths_skips_directories():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
-    flexmock(module.borgmatic.actions.create).should_receive(
-        'process_source_directories'
-    ).and_return(['foo', 'bar'])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+        [Pattern('foo'), Pattern('bar')]
+    )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        source_directories=['foo', 'bar'],
+        patterns=[Pattern('foo'), Pattern('bar')],
         local_borg_version=object,
         global_arguments=object,
         borgmatic_runtime_directory='/run/borgmatic',
@@ -698,7 +723,7 @@ def test_collect_spot_check_source_paths_skips_directories():
         remote_path=object,
         list_files=True,
         stream_processes=True,
-    ).and_return((('borg', 'create'), ('repo::archive',), flexmock(), flexmock()))
+    ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
     flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
         flexmock()
     )
@@ -725,15 +750,16 @@ def test_collect_spot_check_source_paths_skips_directories():
     )
 
 
-def test_collect_spot_check_archive_paths_excludes_directories():
+def test_collect_spot_check_archive_paths_excludes_directories_and_pipes():
     flexmock(module.borgmatic.config.paths).should_receive(
         'get_borgmatic_source_directory'
     ).and_return('/home/user/.borgmatic')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         (
-            'f /etc/path',
-            'f /etc/other',
-            'd /etc/dir',
+            'f etc/path',
+            'p var/pipe',
+            'f etc/other',
+            'd etc/dir',
         )
     )
 
@@ -746,7 +772,7 @@ def test_collect_spot_check_archive_paths_excludes_directories():
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/user/1001/borgmatic',
-    ) == ('/etc/path', '/etc/other')
+    ) == ('etc/path', 'etc/other')
 
 
 def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory_as_stored_with_prefix_truncation():
@@ -755,8 +781,8 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_dir
     ).and_return('/root/.borgmatic')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         (
-            'f /etc/path',
-            'f /borgmatic/some/thing',
+            'f etc/path',
+            'f borgmatic/some/thing',
         )
     )
 
@@ -769,7 +795,7 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_dir
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/user/0/borgmatic',
-    ) == ('/etc/path',)
+    ) == ('etc/path',)
 
 
 def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_directory():
@@ -778,8 +804,8 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_dire
     ).and_return('/root/.borgmatic')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         (
-            'f /etc/path',
-            'f /root/.borgmatic/some/thing',
+            'f etc/path',
+            'f root/.borgmatic/some/thing',
         )
     )
 
@@ -792,7 +818,7 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_dire
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/user/0/borgmatic',
-    ) == ('/etc/path',)
+    ) == ('etc/path',)
 
 
 def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory():
@@ -801,8 +827,8 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_dir
     ).and_return('/root.borgmatic')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         (
-            'f /etc/path',
-            'f /run/user/0/borgmatic/some/thing',
+            'f etc/path',
+            'f run/user/0/borgmatic/some/thing',
         )
     )
 
@@ -815,21 +841,27 @@ def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_dir
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/user/0/borgmatic',
-    ) == ('/etc/path',)
+    ) == ('etc/path',)
 
 
 def test_collect_spot_check_source_paths_uses_working_directory():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
-    flexmock(module.borgmatic.actions.create).should_receive(
-        'process_source_directories'
-    ).and_return(['foo', 'bar'])
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
+        flexmock()
+    )
+    flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
+        [Pattern('foo'), Pattern('bar')]
+    )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        source_directories=['foo', 'bar'],
+        patterns=[Pattern('foo'), Pattern('bar')],
         local_borg_version=object,
         global_arguments=object,
         borgmatic_runtime_directory='/run/borgmatic',
@@ -837,7 +869,7 @@ def test_collect_spot_check_source_paths_uses_working_directory():
         remote_path=object,
         list_files=True,
         stream_processes=True,
-    ).and_return((('borg', 'create'), ('repo::archive',), flexmock(), flexmock()))
+    ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
     flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
         flexmock()
     )
@@ -877,7 +909,7 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
         'hash1  /foo\nhash2  /bar'
     )
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
-        ['hash1 /foo', 'nothash2 /bar']
+        ['hash1 foo', 'nothash2 bar']
     )
 
     assert module.compare_spot_check_hashes(
@@ -904,6 +936,47 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
     ) == ('/bar',)
 
 
+def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes():
+    flexmock(module.random).should_receive('sample').replace_with(
+        lambda population, count: population[:count]
+    )
+    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
+        None,
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_return(
+        'hash1  foo\nhash2  bar'
+    )
+    flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
+        ['hash1 foo', 'nothash2 bar']
+    )
+
+    assert module.compare_spot_check_hashes(
+        repository={'path': 'repo'},
+        archive='archive',
+        config={
+            'checks': [
+                {
+                    'name': 'archives',
+                    'frequency': '2 weeks',
+                },
+                {
+                    'name': 'spot',
+                    'data_sample_percentage': 50,
+                },
+            ]
+        },
+        local_borg_version=flexmock(),
+        global_arguments=flexmock(),
+        local_path=flexmock(),
+        remote_path=flexmock(),
+        log_prefix='repo',
+        source_paths=('foo', 'bar', 'baz', 'quux'),
+    ) == ('bar',)
+
+
 def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
     flexmock(module.random).should_receive('sample').replace_with(
         lambda population, count: population[:count]
@@ -918,7 +991,7 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
         'hash1  /foo\nhash2  /bar'
     )
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
-        ['nothash1 /foo', 'nothash2 /bar']
+        ['nothash1 foo', 'nothash2 bar']
     )
 
     assert module.compare_spot_check_hashes(
@@ -959,7 +1032,7 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
         'hash1  /foo\nhash2  /bar'
     )
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
-        ['hash1 /foo', 'nothash2 /bar']
+        ['hash1 foo', 'nothash2 bar']
     )
 
     assert module.compare_spot_check_hashes(
@@ -997,7 +1070,7 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma
         'hash1  /foo\nhash2  /bar'
     )
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
-        ['hash1 /foo']
+        ['hash1 foo']
     )
 
     assert module.compare_spot_check_hashes(
@@ -1033,7 +1106,7 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching()
         'execute_command_and_capture_output'
     ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1  /foo')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
-        ['hash1 /foo', 'hash2 /bar']
+        ['hash1 foo', 'hash2 bar']
     )
 
     assert module.compare_spot_check_hashes(
@@ -1076,8 +1149,8 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in
         'hash3  /baz\nhash4  /quux'
     )
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
-        ['hash1 /foo', 'hash2 /bar']
-    ).and_return(['hash3 /baz', 'nothash4 /quux'])
+        ['hash1 foo', 'hash2 bar']
+    ).and_return(['hash3 baz', 'nothash4 quux'])
 
     assert module.compare_spot_check_hashes(
         repository={'path': 'repo'},
@@ -1178,7 +1251,7 @@ def test_spot_check_without_any_configuration_errors():
         )
 
 
-def test_spot_check_data_tolerance_percenatge_greater_than_data_sample_percentage_errors():
+def test_spot_check_data_tolerance_percentage_greater_than_data_sample_percentage_errors():
     with pytest.raises(ValueError):
         module.spot_check(
             repository={'path': 'repo'},
@@ -1296,6 +1369,35 @@ def test_spot_check_with_high_enough_tolerances_does_not_raise():
     )
 
 
+def test_spot_check_without_any_source_paths_errors():
+    flexmock(module).should_receive('collect_spot_check_source_paths').and_return(())
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+    flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
+    flexmock(module).should_receive('compare_spot_check_hashes').never()
+
+    with pytest.raises(ValueError):
+        module.spot_check(
+            repository={'path': 'repo'},
+            config={
+                'checks': [
+                    {
+                        'name': 'spot',
+                        'count_tolerance_percentage': 10,
+                        'data_tolerance_percentage': 40,
+                        'data_sample_percentage': 50,
+                    },
+                ]
+            },
+            local_borg_version=flexmock(),
+            global_arguments=flexmock(),
+            local_path=flexmock(),
+            remote_path=flexmock(),
+            borgmatic_runtime_directory='/run/borgmatic',
+        )
+
+
 def test_run_check_checks_archives_for_configured_repository():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()

tests/unit/actions/test_create.py

@@ -1,7 +1,131 @@
+import io
+import sys
+
 import pytest
 from flexmock import flexmock
 
 from borgmatic.actions import create as module
+from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
+
+
+@pytest.mark.parametrize(
+    'pattern_line,expected_pattern',
+    (
+        ('R /foo', Pattern('/foo')),
+        ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)),
+        ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)),
+        ('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)),
+    ),
+)
+def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern):
+    module.parse_pattern(pattern_line) == expected_pattern
+
+
+def test_parse_pattern_with_invalid_pattern_line_errors():
+    with pytest.raises(ValueError):
+        module.parse_pattern('/foo')
+
+
+def test_collect_patterns_converts_source_directories():
+    assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == (
+        Pattern('/foo'),
+        Pattern('/bar'),
+    )
+
+
+def test_collect_patterns_parses_config_patterns():
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /foo', Pattern_style.SHELL
+    ).and_return(Pattern('/foo'))
+    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /bar', Pattern_style.SHELL
+    ).and_return(Pattern('/bar'))
+
+    assert module.collect_patterns({'patterns': ['R /foo', '# comment', '', '   ', 'R /bar']}) == (
+        Pattern('/foo'),
+        Pattern('/bar'),
+    )
+
+
+def test_collect_patterns_converts_exclude_patterns():
+    assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
+        Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
+        Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
+        Pattern('**/baz', Pattern_type.EXCLUDE, Pattern_style.SHELL),
+    )
+
+
+def test_collect_patterns_reads_config_patterns_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('R /foo'))
+    builtins.should_receive('open').with_args('file2.txt').and_return(
+        io.StringIO('R /bar\n# comment\n\n   \nR /baz')
+    )
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /foo', Pattern_style.SHELL
+    ).and_return(Pattern('/foo'))
+    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /bar', Pattern_style.SHELL
+    ).and_return(Pattern('/bar'))
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /baz', Pattern_style.SHELL
+    ).and_return(Pattern('/baz'))
+
+    assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}) == (
+        Pattern('/foo'),
+        Pattern('/bar'),
+        Pattern('/baz'),
+    )
+
+
+def test_collect_patterns_errors_on_missing_config_patterns_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_raise(FileNotFoundError)
+    flexmock(module).should_receive('parse_pattern').never()
+
+    with pytest.raises(ValueError):
+        module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']})
+
+
+def test_collect_patterns_reads_config_exclude_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('/foo'))
+    builtins.should_receive('open').with_args('file2.txt').and_return(
+        io.StringIO('/bar\n# comment\n\n   \n/baz')
+    )
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '- /foo', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '- /bar', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '- /baz', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/baz', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+
+    assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) == (
+        Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
+        Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
+        Pattern('/baz', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
+    )
+
+
+def test_collect_patterns_errors_on_missing_config_exclude_from_file():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('file1.txt').and_raise(OSError)
+    flexmock(module).should_receive('parse_pattern').never()
+
+    with pytest.raises(ValueError):
+        module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']})
 
 
 def test_expand_directory_with_basic_path_passes_it_through():
@@ -22,16 +146,16 @@ def test_expand_directory_with_glob_expands():
     assert paths == ['foo', 'food']
 
 
-def test_expand_directory_with_working_directory_passes_it_through():
+def test_expand_directory_strips_off_working_directory():
     flexmock(module.os.path).should_receive('expanduser').and_return('foo')
     flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo').and_return([]).once()
 
     paths = module.expand_directory('foo', working_directory='/working/dir')
 
-    assert paths == ['/working/dir/foo']
+    assert paths == ['foo']
 
 
-def test_expand_directory_with_glob_passes_through_working_directory():
+def test_expand_directory_globs_working_directory_and_strips_it_off():
     flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
     flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo*').and_return(
         ['/working/dir/foo', '/working/dir/food']
@@ -39,10 +163,32 @@ def test_expand_directory_with_glob_passes_through_working_directory():
 
     paths = module.expand_directory('foo*', working_directory='/working/dir')
 
-    assert paths == ['/working/dir/foo', '/working/dir/food']
+    assert paths == ['foo', 'food']
 
 
-def test_expand_directories_flattens_expanded_directories():
+def test_expand_directory_with_slashdot_hack_globs_working_directory_and_strips_it_off():
+    flexmock(module.os.path).should_receive('expanduser').and_return('./foo*')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/./foo*').and_return(
+        ['/working/dir/./foo', '/working/dir/./food']
+    ).once()
+
+    paths = module.expand_directory('./foo*', working_directory='/working/dir')
+
+    assert paths == ['./foo', './food']
+
+
+def test_expand_directory_with_working_directory_matching_start_of_directory_does_not_strip_it_off():
+    flexmock(module.os.path).should_receive('expanduser').and_return('/working/dir/foo')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo').and_return(
+        ['/working/dir/foo']
+    ).once()
+
+    paths = module.expand_directory('/working/dir/foo', working_directory='/working/dir')
+
+    assert paths == ['/working/dir/foo']
+
+
+def test_expand_patterns_flattens_expanded_directories():
     flexmock(module).should_receive('expand_directory').with_args('~/foo', None).and_return(
         ['/root/foo']
     )
@@ -50,148 +196,224 @@ def test_expand_directories_flattens_expanded_directories():
         ['bar', 'barf']
     )
 
-    paths = module.expand_directories(('~/foo', 'bar*'))
+    paths = module.expand_patterns((Pattern('~/foo'), Pattern('bar*')))
 
-    assert paths == ('/root/foo', 'bar', 'barf')
+    assert paths == (Pattern('/root/foo'), Pattern('bar'), Pattern('barf'))
 
 
-def test_expand_directories_with_working_directory_passes_it_through():
+def test_expand_patterns_with_working_directory_passes_it_through():
     flexmock(module).should_receive('expand_directory').with_args('foo', '/working/dir').and_return(
         ['/working/dir/foo']
     )
 
-    paths = module.expand_directories(('foo',), working_directory='/working/dir')
+    patterns = module.expand_patterns((Pattern('foo'),), working_directory='/working/dir')
 
-    assert paths == ('/working/dir/foo',)
+    assert patterns == (Pattern('/working/dir/foo'),)
 
 
-def test_expand_directories_considers_none_as_no_directories():
-    paths = module.expand_directories(None, None)
+def test_expand_patterns_does_not_expand_skip_paths():
+    flexmock(module).should_receive('expand_directory').with_args('/foo', None).and_return(['/foo'])
+    flexmock(module).should_receive('expand_directory').with_args('/bar*', None).never()
 
-    assert paths == ()
+    patterns = module.expand_patterns((Pattern('/foo'), Pattern('/bar*')), skip_paths=('/bar*',))
+
+    assert patterns == (Pattern('/foo'), Pattern('/bar*'))
 
 
-def test_map_directories_to_devices_gives_device_id_per_path():
+def test_expand_patterns_considers_none_as_no_patterns():
+    assert module.expand_patterns(None) == ()
+
+
+def test_expand_patterns_only_considers_root_patterns():
+    flexmock(module).should_receive('expand_directory').with_args('~/foo', None).and_return(
+        ['/root/foo']
+    )
+    flexmock(module).should_receive('expand_directory').with_args('bar*', None).never()
+
+    paths = module.expand_patterns((Pattern('~/foo'), Pattern('bar*', Pattern_type.INCLUDE)))
+
+    assert paths == (Pattern('/root/foo'), Pattern('bar*', Pattern_type.INCLUDE))
+
+
+def test_device_map_patterns_gives_device_id_per_path():
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
     flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=66))
 
-    device_map = module.map_directories_to_devices(('/foo', '/bar'))
+    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar')))
 
-    assert device_map == {
-        '/foo': 55,
-        '/bar': 66,
-    }
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('/bar', device=66),
+    )
 
 
-def test_map_directories_to_devices_with_missing_path_does_not_error():
+def test_device_map_patterns_only_considers_root_patterns():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar*').never()
+
+    device_map = module.device_map_patterns(
+        (Pattern('/foo'), Pattern('/bar*', Pattern_type.INCLUDE))
+    )
+
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('/bar*', Pattern_type.INCLUDE),
+    )
+
+
+def test_device_map_patterns_with_missing_path_does_not_error():
     flexmock(module.os.path).should_receive('exists').and_return(True).and_return(False)
     flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
     flexmock(module.os).should_receive('stat').with_args('/bar').never()
 
-    device_map = module.map_directories_to_devices(('/foo', '/bar'))
+    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar')))
 
-    assert device_map == {
-        '/foo': 55,
-        '/bar': None,
-    }
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('/bar'),
+    )
 
 
-def test_map_directories_to_devices_uses_working_directory_to_construct_path():
+def test_device_map_patterns_uses_working_directory_to_construct_path():
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
     flexmock(module.os).should_receive('stat').with_args('/working/dir/bar').and_return(
         flexmock(st_dev=66)
     )
 
-    device_map = module.map_directories_to_devices(
-        ('/foo', 'bar'), working_directory='/working/dir'
+    device_map = module.device_map_patterns(
+        (Pattern('/foo'), Pattern('bar')), working_directory='/working/dir'
     )
 
-    assert device_map == {
-        '/foo': 55,
-        'bar': 66,
-    }
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('bar', device=66),
+    )
+
+
+def test_device_map_patterns_with_existing_device_id_does_not_overwrite_it():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=100))
+
+    device_map = module.device_map_patterns((Pattern('/foo'), Pattern('/bar', device=66)))
+
+    assert device_map == (
+        Pattern('/foo', device=55),
+        Pattern('/bar', device=66),
+    )
 
 
 @pytest.mark.parametrize(
-    'directories,additional_directories,expected_directories',
+    'patterns,expected_patterns',
     (
-        ({'/': 1, '/root': 1}, {}, ['/']),
-        ({'/': 1, '/root/': 1}, {}, ['/']),
-        ({'/': 1, '/root': 2}, {}, ['/', '/root']),
-        ({'/root': 1, '/': 1}, {}, ['/']),
-        ({'/root': 1, '/root/foo': 1}, {}, ['/root']),
-        ({'/root/': 1, '/root/foo': 1}, {}, ['/root/']),
-        ({'/root': 1, '/root/foo/': 1}, {}, ['/root']),
-        ({'/root': 1, '/root/foo': 2}, {}, ['/root', '/root/foo']),
-        ({'/root/foo': 1, '/root': 1}, {}, ['/root']),
-        ({'/root': None, '/root/foo': None}, {}, ['/root', '/root/foo']),
-        ({'/root': 1, '/etc': 1, '/root/foo/bar': 1}, {}, ['/etc', '/root']),
-        ({'/root': 1, '/root/foo': 1, '/root/foo/bar': 1}, {}, ['/root']),
-        ({'/dup': 1, '/dup': 1}, {}, ['/dup']),
-        ({'/foo': 1, '/bar': 1}, {}, ['/bar', '/foo']),
-        ({'/foo': 1, '/bar': 2}, {}, ['/bar', '/foo']),
-        ({'/root/foo': 1}, {'/root': 1}, []),
-        ({'/root/foo': 1}, {'/root': 2}, ['/root/foo']),
-        ({'/root/foo': 1}, {}, ['/root/foo']),
+        ((Pattern('/', device=1), Pattern('/root', device=1)), (Pattern('/', device=1),)),
+        ((Pattern('/', device=1), Pattern('/root/', device=1)), (Pattern('/', device=1),)),
+        (
+            (Pattern('/', device=1), Pattern('/root', device=2)),
+            (Pattern('/', device=1), Pattern('/root', device=2)),
+        ),
+        ((Pattern('/root', device=1), Pattern('/', device=1)), (Pattern('/', device=1),)),
+        (
+            (Pattern('/root', device=1), Pattern('/root/foo', device=1)),
+            (Pattern('/root', device=1),),
+        ),
+        (
+            (Pattern('/root/', device=1), Pattern('/root/foo', device=1)),
+            (Pattern('/root/', device=1),),
+        ),
+        (
+            (Pattern('/root', device=1), Pattern('/root/foo/', device=1)),
+            (Pattern('/root', device=1),),
+        ),
+        (
+            (Pattern('/root', device=1), Pattern('/root/foo', device=2)),
+            (Pattern('/root', device=1), Pattern('/root/foo', device=2)),
+        ),
+        (
+            (Pattern('/root/foo', device=1), Pattern('/root', device=1)),
+            (Pattern('/root', device=1),),
+        ),
+        (
+            (Pattern('/root', device=None), Pattern('/root/foo', device=None)),
+            (Pattern('/root'), Pattern('/root/foo')),
+        ),
+        (
+            (
+                Pattern('/root', device=1),
+                Pattern('/etc', device=1),
+                Pattern('/root/foo/bar', device=1),
+            ),
+            (Pattern('/root', device=1), Pattern('/etc', device=1)),
+        ),
+        (
+            (
+                Pattern('/root', device=1),
+                Pattern('/root/foo', device=1),
+                Pattern('/root/foo/bar', device=1),
+            ),
+            (Pattern('/root', device=1),),
+        ),
+        ((Pattern('/dup', device=1), Pattern('/dup', device=1)), (Pattern('/dup', device=1),)),
+        (
+            (Pattern('/foo', device=1), Pattern('/bar', device=1)),
+            (Pattern('/foo', device=1), Pattern('/bar', device=1)),
+        ),
+        (
+            (Pattern('/foo', device=1), Pattern('/bar', device=2)),
+            (Pattern('/foo', device=1), Pattern('/bar', device=2)),
+        ),
+        ((Pattern('/root/foo', device=1),), (Pattern('/root/foo', device=1),)),
+        (
+            (Pattern('/', device=1), Pattern('/root', Pattern_type.INCLUDE, device=1)),
+            (Pattern('/', device=1), Pattern('/root', Pattern_type.INCLUDE, device=1)),
+        ),
+        (
+            (Pattern('/root', Pattern_type.INCLUDE, device=1), Pattern('/', device=1)),
+            (Pattern('/root', Pattern_type.INCLUDE, device=1), Pattern('/', device=1)),
+        ),
     ),
 )
-def test_deduplicate_directories_removes_child_paths_on_the_same_filesystem(
-    directories, additional_directories, expected_directories
-):
-    assert (
-        module.deduplicate_directories(directories, additional_directories) == expected_directories
+def test_deduplicate_patterns_omits_child_paths_on_the_same_filesystem(patterns, expected_patterns):
+    assert module.deduplicate_patterns(patterns) == expected_patterns
+
+
+def test_process_patterns_includes_patterns():
+    flexmock(module).should_receive('deduplicate_patterns').and_return(
+        (Pattern('foo'), Pattern('bar'))
     )
-
-
-def test_pattern_root_directories_deals_with_none_patterns():
-    assert module.pattern_root_directories(patterns=None) == []
-
-
-def test_pattern_root_directories_parses_roots_and_ignores_others():
-    assert module.pattern_root_directories(
-        ['R /root', '+ /root/foo', '- /root/foo/bar', 'R /baz']
-    ) == ['/root', '/baz']
-
-
-def test_process_source_directories_includes_source_directories():
-    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
-        '/working'
-    )
-    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
-    flexmock(module).should_receive('map_directories_to_devices').and_return({})
-    flexmock(module).should_receive('expand_directories').with_args(
-        ('foo', 'bar'), working_directory='/working'
+    flexmock(module).should_receive('device_map_patterns').and_return({})
+    flexmock(module).should_receive('expand_patterns').with_args(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+        skip_paths=set(),
     ).and_return(()).once()
-    flexmock(module).should_receive('pattern_root_directories').and_return(())
-    flexmock(module).should_receive('expand_directories').with_args(
-        (), working_directory='/working'
-    ).and_return(())
 
-    assert module.process_source_directories(
-        config={'source_directories': ['foo', 'bar']},
-    ) == ('foo', 'bar')
+    assert module.process_patterns(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+    ) == [Pattern('foo'), Pattern('bar')]
 
 
-def test_process_source_directories_prefers_source_directory_argument_to_config():
-    flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
-        '/working'
+def test_process_patterns_skips_expand_for_requested_paths():
+    skip_paths = {flexmock()}
+    flexmock(module).should_receive('deduplicate_patterns').and_return(
+        (Pattern('foo'), Pattern('bar'))
     )
-    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
-    flexmock(module).should_receive('map_directories_to_devices').and_return({})
-    flexmock(module).should_receive('expand_directories').with_args(
-        ('foo', 'bar'), working_directory='/working'
+    flexmock(module).should_receive('device_map_patterns').and_return({})
+    flexmock(module).should_receive('expand_patterns').with_args(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+        skip_paths=skip_paths,
     ).and_return(()).once()
-    flexmock(module).should_receive('pattern_root_directories').and_return(())
-    flexmock(module).should_receive('expand_directories').with_args(
-        (), working_directory='/working'
-    ).and_return(())
 
-    assert module.process_source_directories(
-        config={'source_directories': ['nope']},
-        source_directories=['foo', 'bar'],
-    ) == ('foo', 'bar')
+    assert module.process_patterns(
+        (Pattern('foo'), Pattern('bar')),
+        working_directory='/working',
+        skip_expand_paths=skip_paths,
+    ) == [Pattern('foo'), Pattern('bar')]
 
 
 def test_run_create_executes_and_calls_hooks_for_configured_repository():
@@ -206,7 +428,8 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
-    flexmock(module).should_receive('process_source_directories').and_return([])
+    flexmock(module).should_receive('collect_patterns').and_return(())
+    flexmock(module).should_receive('process_patterns').and_return([])
     flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=None,
@@ -248,7 +471,8 @@ def test_run_create_runs_with_selected_repository():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
-    flexmock(module).should_receive('process_source_directories').and_return([])
+    flexmock(module).should_receive('collect_patterns').and_return(())
+    flexmock(module).should_receive('process_patterns').and_return([])
     flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=flexmock(),
@@ -327,7 +551,8 @@ def test_run_create_produces_json():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
-    flexmock(module).should_receive('process_source_directories').and_return([])
+    flexmock(module).should_receive('collect_patterns').and_return(())
+    flexmock(module).should_receive('process_patterns').and_return([])
     flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=flexmock(),

tests/unit/hooks/data_source/test_bootstrap.py

@@ -22,12 +22,12 @@ def test_dump_data_sources_creates_manifest_file():
     ).once()
 
     module.dump_data_sources(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     )
 
@@ -43,7 +43,7 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes
         log_prefix='test',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=True,
     )
 
@@ -53,12 +53,12 @@ def test_dump_data_sources_with_dry_run_does_not_create_manifest_file():
     flexmock(module.json).should_receive('dump').never()
 
     module.dump_data_sources(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=True,
     )
 
@@ -74,7 +74,7 @@ def test_remove_data_source_dumps_deletes_manifest_and_parent_directory():
     flexmock(module.os).should_receive('rmdir').with_args('/run/borgmatic/bootstrap').once()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',
@@ -91,7 +91,7 @@ def test_remove_data_source_dumps_with_dry_run_bails():
     flexmock(module.os).should_receive('rmdir').never()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',
@@ -110,7 +110,7 @@ def test_remove_data_source_dumps_swallows_manifest_file_not_found_error():
     flexmock(module.os).should_receive('rmdir').with_args('/run/borgmatic/bootstrap').once()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',
@@ -131,7 +131,7 @@ def test_remove_data_source_dumps_swallows_manifest_parent_directory_not_found_e
     ).once()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',

tests/unit/hooks/data_source/test_btrfs.py

@@ -1,5 +1,7 @@
+import pytest
 from flexmock import flexmock
 
+from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
 from borgmatic.hooks.data_source import btrfs as module
 
 
@@ -7,13 +9,46 @@ def test_get_filesystem_mount_points_parses_findmnt_output():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
     ).and_return(
-        '/mnt0   /dev/loop0 btrfs  rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/\n'
-        '/mnt1   /dev/loop1 btrfs  rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/\n'
+        '''{
+           "filesystems": [
+              {
+                 "target": "/mnt0",
+                 "source": "/dev/loop0",
+                 "fstype": "btrfs",
+                 "options": "rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/"
+              },
+              {
+                 "target": "/mnt1",
+                 "source": "/dev/loop0",
+                 "fstype": "btrfs",
+                 "options": "rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/"
+              }
+           ]
+        }
+        '''
     )
 
     assert module.get_filesystem_mount_points('findmnt') == ('/mnt0', '/mnt1')
 
 
+def test_get_filesystem_mount_points_with_invalid_findmnt_json_errors():
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).and_return('{')
+
+    with pytest.raises(ValueError):
+        module.get_filesystem_mount_points('findmnt')
+
+
+def test_get_filesystem_mount_points_with_findmnt_json_missing_filesystems_errors():
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).and_return('{"wtf": "something is wrong here"}')
+
+    with pytest.raises(ValueError):
+        module.get_filesystem_mount_points('findmnt')
+
+
 def test_get_subvolumes_for_filesystem_parses_subvolume_list_output():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
@@ -46,7 +81,7 @@ def test_get_subvolumes_for_filesystem_skips_empty_filesystem_mount_points():
     assert module.get_subvolumes_for_filesystem('btrfs', ' ') == ()
 
 
-def test_get_subvolumes_collects_subvolumes_matching_source_directories_from_all_filesystems():
+def test_get_subvolumes_collects_subvolumes_matching_patterns_from_all_filesystems():
     flexmock(module).should_receive('get_filesystem_mount_points').and_return(('/mnt1', '/mnt2'))
     flexmock(module).should_receive('get_subvolumes_for_filesystem').with_args(
         'btrfs', '/mnt1'
@@ -57,22 +92,31 @@ def test_get_subvolumes_collects_subvolumes_matching_source_directories_from_all
 
     for path in ('/one', '/four'):
         flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-            'get_contained_directories'
-        ).with_args(path, object).and_return((path,))
+            'get_contained_patterns'
+        ).with_args(path, object).and_return((Pattern(path),))
     for path in ('/two', '/three'):
         flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-            'get_contained_directories'
+            'get_contained_patterns'
         ).with_args(path, object).and_return(())
 
     assert module.get_subvolumes(
-        'btrfs', 'findmnt', source_directories=['/one', '/four', '/five', '/six', '/mnt2', '/mnt3']
+        'btrfs',
+        'findmnt',
+        patterns=[
+            Pattern('/one'),
+            Pattern('/four'),
+            Pattern('/five'),
+            Pattern('/six'),
+            Pattern('/mnt2'),
+            Pattern('/mnt3'),
+        ],
     ) == (
-        module.Subvolume('/four', contained_source_directories=('/four',)),
-        module.Subvolume('/one', contained_source_directories=('/one',)),
+        module.Subvolume('/four', contained_patterns=(Pattern('/four'),)),
+        module.Subvolume('/one', contained_patterns=(Pattern('/one'),)),
     )
 
 
-def test_get_subvolumes_without_source_directories_collects_all_subvolumes_from_all_filesystems():
+def test_get_subvolumes_without_patterns_collects_all_subvolumes_from_all_filesystems():
     flexmock(module).should_receive('get_filesystem_mount_points').and_return(('/mnt1', '/mnt2'))
     flexmock(module).should_receive('get_subvolumes_for_filesystem').with_args(
         'btrfs', '/mnt1'
@@ -83,24 +127,78 @@ def test_get_subvolumes_without_source_directories_collects_all_subvolumes_from_
 
     for path in ('/one', '/two', '/three', '/four'):
         flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-            'get_contained_directories'
-        ).with_args(path, object).and_return((path,))
+            'get_contained_patterns'
+        ).with_args(path, object).and_return((Pattern(path),))
 
     assert module.get_subvolumes('btrfs', 'findmnt') == (
-        module.Subvolume('/four', contained_source_directories=('/four',)),
-        module.Subvolume('/one', contained_source_directories=('/one',)),
-        module.Subvolume('/three', contained_source_directories=('/three',)),
-        module.Subvolume('/two', contained_source_directories=('/two',)),
+        module.Subvolume('/four', contained_patterns=(Pattern('/four'),)),
+        module.Subvolume('/one', contained_patterns=(Pattern('/one'),)),
+        module.Subvolume('/three', contained_patterns=(Pattern('/three'),)),
+        module.Subvolume('/two', contained_patterns=(Pattern('/two'),)),
     )
 
 
-def test_dump_data_sources_snapshots_each_subvolume_and_updates_source_directories():
-    source_directories = ['/foo', '/mnt/subvol1']
+@pytest.mark.parametrize(
+    'subvolume_path,expected_snapshot_path',
+    (
+        ('/foo/bar', '/foo/bar/.borgmatic-snapshot-1234/foo/bar'),
+        ('/', '/.borgmatic-snapshot-1234'),
+    ),
+)
+def test_make_snapshot_path_includes_stripped_subvolume_path(
+    subvolume_path, expected_snapshot_path
+):
+    flexmock(module.os).should_receive('getpid').and_return(1234)
+
+    assert module.make_snapshot_path(subvolume_path) == expected_snapshot_path
+
+
+@pytest.mark.parametrize(
+    'subvolume_path,pattern,expected_pattern',
+    (
+        (
+            '/foo/bar',
+            Pattern('/foo/bar/baz'),
+            Pattern('/foo/bar/.borgmatic-snapshot-1234/./foo/bar/baz'),
+        ),
+        ('/foo/bar', Pattern('/foo/bar'), Pattern('/foo/bar/.borgmatic-snapshot-1234/./foo/bar')),
+        (
+            '/foo/bar',
+            Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
+            Pattern(
+                '^/foo/bar/.borgmatic-snapshot-1234/./foo/bar',
+                Pattern_type.INCLUDE,
+                Pattern_style.REGULAR_EXPRESSION,
+            ),
+        ),
+        (
+            '/foo/bar',
+            Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
+            Pattern(
+                '/foo/bar/.borgmatic-snapshot-1234/./foo/bar',
+                Pattern_type.INCLUDE,
+                Pattern_style.REGULAR_EXPRESSION,
+            ),
+        ),
+        ('/', Pattern('/foo'), Pattern('/.borgmatic-snapshot-1234/./foo')),
+        ('/', Pattern('/'), Pattern('/.borgmatic-snapshot-1234/./')),
+    ),
+)
+def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
+    subvolume_path, pattern, expected_pattern
+):
+    flexmock(module.os).should_receive('getpid').and_return(1234)
+
+    assert module.make_borg_snapshot_pattern(subvolume_path, pattern) == expected_pattern
+
+
+def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns():
+    patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')]
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(
         (
-            module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),
-            module.Subvolume('/mnt/subvol2', contained_source_directories=('/mnt/subvol2',)),
+            module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),
+            module.Subvolume('/mnt/subvol2', contained_patterns=(Pattern('/mnt/subvol2'),)),
         )
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
@@ -115,18 +213,30 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_source_directori
     flexmock(module).should_receive('snapshot_subvolume').with_args(
         'btrfs', '/mnt/subvol2', '/mnt/subvol2/.borgmatic-1234/mnt/subvol2'
     ).once()
-    flexmock(module).should_receive('make_snapshot_exclude_path').with_args(
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').with_args(
         '/mnt/subvol1'
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234')
-    flexmock(module).should_receive('make_snapshot_exclude_path').with_args(
+    ).and_return(
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        )
+    )
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').with_args(
         '/mnt/subvol2'
-    ).and_return('/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234')
-    flexmock(module).should_receive('make_borg_source_directory_path').with_args(
+    ).and_return(
+        Pattern(
+            '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        )
+    )
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
         '/mnt/subvol1', object
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1')
-    flexmock(module).should_receive('make_borg_source_directory_path').with_args(
+    ).and_return(Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'))
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
         '/mnt/subvol2', object
-    ).and_return('/mnt/subvol2/.borgmatic-1234/mnt/subvol2')
+    ).and_return(Pattern('/mnt/subvol2/.borgmatic-1234/mnt/subvol2'))
 
     assert (
         module.dump_data_sources(
@@ -135,31 +245,37 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_source_directori
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/foo',
-        '/mnt/subvol1/.borgmatic-1234/mnt/subvol1',
-        '/mnt/subvol2/.borgmatic-1234/mnt/subvol2',
+    assert patterns == [
+        Pattern('/foo'),
+        Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        ),
+        Pattern('/mnt/subvol2/.borgmatic-1234/mnt/subvol2'),
+        Pattern(
+            '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        ),
     ]
     assert config == {
         'btrfs': {},
-        'exclude_patterns': [
-            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
-        ],
     }
 
 
 def test_dump_data_sources_uses_custom_btrfs_command_in_commands():
-    source_directories = ['/foo', '/mnt/subvol1']
+    patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')]
     config = {'btrfs': {'btrfs_command': '/usr/local/bin/btrfs'}}
     flexmock(module).should_receive('get_subvolumes').and_return(
-        (module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),)
+        (module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),)
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
         '/mnt/subvol1/.borgmatic-1234/mnt/subvol1'
@@ -167,12 +283,18 @@ def test_dump_data_sources_uses_custom_btrfs_command_in_commands():
     flexmock(module).should_receive('snapshot_subvolume').with_args(
         '/usr/local/bin/btrfs', '/mnt/subvol1', '/mnt/subvol1/.borgmatic-1234/mnt/subvol1'
     ).once()
-    flexmock(module).should_receive('make_snapshot_exclude_path').with_args(
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').with_args(
         '/mnt/subvol1'
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234')
-    flexmock(module).should_receive('make_borg_source_directory_path').with_args(
+    ).and_return(
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        )
+    )
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
         '/mnt/subvol1', object
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1')
+    ).and_return(Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'))
 
     assert (
         module.dump_data_sources(
@@ -181,33 +303,35 @@ def test_dump_data_sources_uses_custom_btrfs_command_in_commands():
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/foo',
-        '/mnt/subvol1/.borgmatic-1234/mnt/subvol1',
+    assert patterns == [
+        Pattern('/foo'),
+        Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        ),
     ]
     assert config == {
         'btrfs': {
             'btrfs_command': '/usr/local/bin/btrfs',
         },
-        'exclude_patterns': [
-            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-        ],
     }
 
 
 def test_dump_data_sources_uses_custom_findmnt_command_in_commands():
-    source_directories = ['/foo', '/mnt/subvol1']
+    patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')]
     config = {'btrfs': {'findmnt_command': '/usr/local/bin/findmnt'}}
     flexmock(module).should_receive('get_subvolumes').with_args(
-        'btrfs', '/usr/local/bin/findmnt', source_directories
+        'btrfs', '/usr/local/bin/findmnt', patterns
     ).and_return(
-        (module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),)
+        (module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),)
     ).once()
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
         '/mnt/subvol1/.borgmatic-1234/mnt/subvol1'
@@ -215,12 +339,18 @@ def test_dump_data_sources_uses_custom_findmnt_command_in_commands():
     flexmock(module).should_receive('snapshot_subvolume').with_args(
         'btrfs', '/mnt/subvol1', '/mnt/subvol1/.borgmatic-1234/mnt/subvol1'
     ).once()
-    flexmock(module).should_receive('make_snapshot_exclude_path').with_args(
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').with_args(
         '/mnt/subvol1'
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234')
-    flexmock(module).should_receive('make_borg_source_directory_path').with_args(
+    ).and_return(
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        )
+    )
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
         '/mnt/subvol1', object
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1')
+    ).and_return(Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'))
 
     assert (
         module.dump_data_sources(
@@ -229,37 +359,39 @@ def test_dump_data_sources_uses_custom_findmnt_command_in_commands():
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/foo',
-        '/mnt/subvol1/.borgmatic-1234/mnt/subvol1',
+    assert patterns == [
+        Pattern('/foo'),
+        Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        ),
     ]
     assert config == {
         'btrfs': {
             'findmnt_command': '/usr/local/bin/findmnt',
         },
-        'exclude_patterns': [
-            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-        ],
     }
 
 
-def test_dump_data_sources_with_dry_run_skips_snapshot_and_source_directories_update():
-    source_directories = ['/foo', '/mnt/subvol1']
+def test_dump_data_sources_with_dry_run_skips_snapshot_and_patterns_update():
+    patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')]
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(
-        (module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),)
+        (module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),)
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
         '/mnt/subvol1/.borgmatic-1234/mnt/subvol1'
     )
     flexmock(module).should_receive('snapshot_subvolume').never()
-    flexmock(module).should_receive('make_snapshot_exclude_path').never()
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').never()
 
     assert (
         module.dump_data_sources(
@@ -268,23 +400,23 @@ def test_dump_data_sources_with_dry_run_skips_snapshot_and_source_directories_up
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=True,
         )
         == []
     )
 
-    assert source_directories == ['/foo', '/mnt/subvol1']
+    assert patterns == [Pattern('/foo'), Pattern('/mnt/subvol1')]
     assert config == {'btrfs': {}}
 
 
-def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_source_directories_update():
-    source_directories = ['/foo', '/mnt/subvol1']
+def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_patterns_update():
+    patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')]
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(())
     flexmock(module).should_receive('make_snapshot_path').never()
     flexmock(module).should_receive('snapshot_subvolume').never()
-    flexmock(module).should_receive('make_snapshot_exclude_path').never()
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').never()
 
     assert (
         module.dump_data_sources(
@@ -293,23 +425,23 @@ def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_source
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == ['/foo', '/mnt/subvol1']
+    assert patterns == [Pattern('/foo'), Pattern('/mnt/subvol1')]
     assert config == {'btrfs': {}}
 
 
 def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns():
-    source_directories = ['/foo', '/mnt/subvol1']
+    patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')]
     config = {'btrfs': {}, 'exclude_patterns': ['/bar']}
     flexmock(module).should_receive('get_subvolumes').and_return(
         (
-            module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),
-            module.Subvolume('/mnt/subvol2', contained_source_directories=('/mnt/subvol2',)),
+            module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),
+            module.Subvolume('/mnt/subvol2', contained_patterns=(Pattern('/mnt/subvol2'),)),
         )
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
@@ -324,18 +456,30 @@ def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns():
     flexmock(module).should_receive('snapshot_subvolume').with_args(
         'btrfs', '/mnt/subvol2', '/mnt/subvol2/.borgmatic-1234/mnt/subvol2'
     ).once()
-    flexmock(module).should_receive('make_snapshot_exclude_path').with_args(
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').with_args(
         '/mnt/subvol1'
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234')
-    flexmock(module).should_receive('make_snapshot_exclude_path').with_args(
+    ).and_return(
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        )
+    )
+    flexmock(module).should_receive('make_snapshot_exclude_pattern').with_args(
         '/mnt/subvol2'
-    ).and_return('/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234')
-    flexmock(module).should_receive('make_borg_source_directory_path').with_args(
+    ).and_return(
+        Pattern(
+            '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        )
+    )
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
         '/mnt/subvol1', object
-    ).and_return('/mnt/subvol1/.borgmatic-1234/mnt/subvol1')
-    flexmock(module).should_receive('make_borg_source_directory_path').with_args(
+    ).and_return(Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'))
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
         '/mnt/subvol2', object
-    ).and_return('/mnt/subvol2/.borgmatic-1234/mnt/subvol2')
+    ).and_return(Pattern('/mnt/subvol2/.borgmatic-1234/mnt/subvol2'))
 
     assert (
         module.dump_data_sources(
@@ -344,24 +488,30 @@ def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns():
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/foo',
-        '/mnt/subvol1/.borgmatic-1234/mnt/subvol1',
-        '/mnt/subvol2/.borgmatic-1234/mnt/subvol2',
+    assert patterns == [
+        Pattern('/foo'),
+        Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
+        Pattern(
+            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        ),
+        Pattern('/mnt/subvol2/.borgmatic-1234/mnt/subvol2'),
+        Pattern(
+            '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
+            Pattern_type.EXCLUDE,
+            Pattern_style.FNMATCH,
+        ),
     ]
     assert config == {
         'btrfs': {},
-        'exclude_patterns': [
-            '/bar',
-            '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
-        ],
+        'exclude_patterns': ['/bar'],
     }
 
 
@@ -369,8 +519,8 @@ def test_remove_data_source_dumps_deletes_snapshots():
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(
         (
-            module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),
-            module.Subvolume('/mnt/subvol2', contained_source_directories=('/mnt/subvol2',)),
+            module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),
+            module.Subvolume('/mnt/subvol2', contained_patterns=(Pattern('/mnt/subvol2'),)),
         )
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
@@ -451,6 +601,24 @@ def test_remove_data_source_dumps_deletes_snapshots():
     )
 
 
+def test_remove_data_source_dumps_without_hook_configuration_bails():
+    flexmock(module).should_receive('get_subvolumes').never()
+    flexmock(module).should_receive('make_snapshot_path').never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'replace_temporary_subdirectory_with_glob'
+    ).never()
+    flexmock(module).should_receive('delete_snapshot').never()
+    flexmock(module.shutil).should_receive('rmtree').never()
+
+    module.remove_data_source_dumps(
+        hook_config=None,
+        config={'source_directories': '/mnt/subvolume'},
+        log_prefix='test',
+        borgmatic_runtime_directory='/run/borgmatic',
+        dry_run=False,
+    )
+
+
 def test_remove_data_source_dumps_with_get_subvolumes_file_not_found_error_bails():
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_raise(FileNotFoundError)
@@ -495,8 +663,8 @@ def test_remove_data_source_dumps_with_dry_run_skips_deletes():
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(
         (
-            module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),
-            module.Subvolume('/mnt/subvol2', contained_source_directories=('/mnt/subvol2',)),
+            module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),
+            module.Subvolume('/mnt/subvol2', contained_patterns=(Pattern('/mnt/subvol2'),)),
         )
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
@@ -578,8 +746,8 @@ def test_remove_data_source_without_snapshots_skips_deletes():
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(
         (
-            module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),
-            module.Subvolume('/mnt/subvol2', contained_source_directories=('/mnt/subvol2',)),
+            module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),
+            module.Subvolume('/mnt/subvol2', contained_patterns=(Pattern('/mnt/subvol2'),)),
         )
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
@@ -622,8 +790,8 @@ def test_remove_data_source_dumps_with_delete_snapshot_file_not_found_error_bail
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(
         (
-            module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),
-            module.Subvolume('/mnt/subvol2', contained_source_directories=('/mnt/subvol2',)),
+            module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),
+            module.Subvolume('/mnt/subvol2', contained_patterns=(Pattern('/mnt/subvol2'),)),
         )
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(
@@ -686,8 +854,8 @@ def test_remove_data_source_dumps_with_delete_snapshot_called_process_error_bail
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_return(
         (
-            module.Subvolume('/mnt/subvol1', contained_source_directories=('/mnt/subvol1',)),
-            module.Subvolume('/mnt/subvol2', contained_source_directories=('/mnt/subvol2',)),
+            module.Subvolume('/mnt/subvol1', contained_patterns=(Pattern('/mnt/subvol1'),)),
+            module.Subvolume('/mnt/subvol2', contained_patterns=(Pattern('/mnt/subvol2'),)),
         )
     )
     flexmock(module).should_receive('make_snapshot_path').with_args('/mnt/subvol1').and_return(

tests/unit/hooks/data_source/test_dump.py

@@ -15,6 +15,13 @@ def test_make_data_source_dump_filename_uses_name_and_hostname():
     )
 
 
+def test_make_data_source_dump_filename_uses_name_and_hostname_and_port():
+    assert (
+        module.make_data_source_dump_filename('databases', 'test', 'hostname', 1234)
+        == 'databases/hostname:1234/test'
+    )
+
+
 def test_make_data_source_dump_filename_without_hostname_defaults_to_localhost():
     assert module.make_data_source_dump_filename('databases', 'test') == 'databases/localhost/test'
 

tests/unit/hooks/data_source/test_lvm.py

@@ -1,10 +1,11 @@
 import pytest
 from flexmock import flexmock
 
+from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
 from borgmatic.hooks.data_source import lvm as module
 
 
-def test_get_logical_volumes_filters_by_source_directories():
+def test_get_logical_volumes_filters_by_patterns():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
     ).and_return(
@@ -36,28 +37,30 @@ def test_get_logical_volumes_filters_by_source_directories():
         }
         '''
     )
-    contained = {'/mnt/lvolume', '/mnt/lvolume/subdir'}
+    contained = {Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')}
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).with_args(None, contained).never()
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
-    ).with_args('/mnt/lvolume', contained).and_return(('/mnt/lvolume', '/mnt/lvolume/subdir'))
+        'get_contained_patterns'
+    ).with_args('/mnt/lvolume', contained).and_return(
+        (Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir'))
+    )
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).with_args('/mnt/other', contained).and_return(())
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).with_args('/mnt/notlvm', contained).never()
 
     assert module.get_logical_volumes(
-        'lsblk', source_directories=('/mnt/lvolume', '/mnt/lvolume/subdir')
+        'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir'))
     ) == (
         module.Logical_volume(
-            'vgroup-lvolume',
-            '/dev/mapper/vgroup-lvolume',
-            '/mnt/lvolume',
-            ('/mnt/lvolume', '/mnt/lvolume/subdir'),
+            name='vgroup-lvolume',
+            device_path='/dev/mapper/vgroup-lvolume',
+            mount_point='/mnt/lvolume',
+            contained_patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')),
         ),
     )
 
@@ -68,12 +71,12 @@ def test_get_logical_volumes_with_invalid_lsblk_json_errors():
     ).and_return('{')
 
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).never()
 
     with pytest.raises(ValueError):
         module.get_logical_volumes(
-            'lsblk', source_directories=('/mnt/lvolume', '/mnt/lvolume/subdir')
+            'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir'))
         )
 
 
@@ -83,12 +86,12 @@ def test_get_logical_volumes_with_lsblk_json_missing_keys_errors():
     ).and_return('{"block_devices": [{}]}')
 
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).never()
 
     with pytest.raises(ValueError):
         module.get_logical_volumes(
-            'lsblk', source_directories=('/mnt/lvolume', '/mnt/lvolume/subdir')
+            'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir'))
         )
 
 
@@ -99,6 +102,8 @@ def test_snapshot_logical_volume_with_percentage_snapshot_name_uses_lvcreate_ext
             '--snapshot',
             '--extents',
             '10%ORIGIN',
+            '--permission',
+            'r',
             '--name',
             'snap',
             '/dev/snap',
@@ -116,6 +121,8 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
             '--snapshot',
             '--size',
             '10TB',
+            '--permission',
+            'r',
             '--name',
             'snap',
             '/dev/snap',
@@ -126,22 +133,56 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
     module.snapshot_logical_volume('lvcreate', 'snap', '/dev/snap', '10TB')
 
 
-def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories():
+@pytest.mark.parametrize(
+    'pattern,expected_pattern',
+    (
+        (
+            Pattern('/foo/bar/baz'),
+            Pattern('/run/borgmatic/lvm_snapshots/./foo/bar/baz'),
+        ),
+        (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/./foo/bar')),
+        (
+            Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
+            Pattern(
+                '^/run/borgmatic/lvm_snapshots/./foo/bar',
+                Pattern_type.INCLUDE,
+                Pattern_style.REGULAR_EXPRESSION,
+            ),
+        ),
+        (
+            Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
+            Pattern(
+                '/run/borgmatic/lvm_snapshots/./foo/bar',
+                Pattern_type.INCLUDE,
+                Pattern_style.REGULAR_EXPRESSION,
+            ),
+        ),
+        (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/./foo')),
+        (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/./')),
+    ),
+)
+def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
+    pattern, expected_pattern
+):
+    assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern
+
+
+def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns():
     config = {'lvm': {}}
-    source_directories = ['/mnt/lvolume1/subdir', '/mnt/lvolume2']
+    patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
     flexmock(module).should_receive('get_logical_volumes').and_return(
         (
             module.Logical_volume(
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -168,6 +209,12 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories()
     flexmock(module).should_receive('mount_snapshot').with_args(
         'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
     ).once()
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume2'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
 
     assert (
         module.dump_data_sources(
@@ -176,21 +223,21 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories()
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir',
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume2',
+    assert patterns == [
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
     ]
 
 
 def test_dump_data_sources_with_no_logical_volumes_skips_snapshots():
     config = {'lvm': {}}
-    source_directories = ['/mnt/lvolume1/subdir', '/mnt/lvolume2']
+    patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
     flexmock(module).should_receive('get_logical_volumes').and_return(())
     flexmock(module).should_receive('snapshot_logical_volume').never()
     flexmock(module).should_receive('mount_snapshot').never()
@@ -202,31 +249,31 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots():
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == ['/mnt/lvolume1/subdir', '/mnt/lvolume2']
+    assert patterns == [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
 
 
 def test_dump_data_sources_uses_snapshot_size_for_snapshot():
     config = {'lvm': {'snapshot_size': '1000PB'}}
-    source_directories = ['/mnt/lvolume1/subdir', '/mnt/lvolume2']
+    patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
     flexmock(module).should_receive('get_logical_volumes').and_return(
         (
             module.Logical_volume(
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -259,6 +306,12 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot():
     flexmock(module).should_receive('mount_snapshot').with_args(
         'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
     ).once()
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume2'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
 
     assert (
         module.dump_data_sources(
@@ -267,15 +320,15 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot():
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir',
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume2',
+    assert patterns == [
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
     ]
 
 
@@ -288,20 +341,20 @@ def test_dump_data_sources_uses_custom_commands():
             'mount_command': '/usr/local/bin/mount',
         },
     }
-    source_directories = ['/mnt/lvolume1/subdir', '/mnt/lvolume2']
+    patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
     flexmock(module).should_receive('get_logical_volumes').and_return(
         (
             module.Logical_volume(
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -334,6 +387,12 @@ def test_dump_data_sources_uses_custom_commands():
     flexmock(module).should_receive('mount_snapshot').with_args(
         '/usr/local/bin/mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
     ).once()
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume2'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
 
     assert (
         module.dump_data_sources(
@@ -342,34 +401,34 @@ def test_dump_data_sources_uses_custom_commands():
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir',
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume2',
+    assert patterns == [
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
     ]
 
 
-def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_source_directories():
+def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patterns():
     config = {'lvm': {}}
-    source_directories = ['/mnt/lvolume1/subdir', '/mnt/lvolume2']
+    patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
     flexmock(module).should_receive('get_logical_volumes').and_return(
         (
             module.Logical_volume(
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -394,34 +453,34 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_sourc
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=True,
         )
         == []
     )
 
-    assert source_directories == [
-        '/mnt/lvolume1/subdir',
-        '/mnt/lvolume2',
+    assert patterns == [
+        Pattern('/mnt/lvolume1/subdir'),
+        Pattern('/mnt/lvolume2'),
     ]
 
 
-def test_dump_data_sources_ignores_mismatch_between_source_directories_and_contained_source_directories():
+def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns():
     config = {'lvm': {}}
-    source_directories = ['/hmm']
+    patterns = [Pattern('/hmm')]
     flexmock(module).should_receive('get_logical_volumes').and_return(
         (
             module.Logical_volume(
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -448,6 +507,12 @@ def test_dump_data_sources_ignores_mismatch_between_source_directories_and_conta
     flexmock(module).should_receive('mount_snapshot').with_args(
         'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
     ).once()
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/lvolume2'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
 
     assert (
         module.dump_data_sources(
@@ -456,35 +521,35 @@ def test_dump_data_sources_ignores_mismatch_between_source_directories_and_conta
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [
-        '/hmm',
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir',
-        '/run/borgmatic/lvm_snapshots/./mnt/lvolume2',
+    assert patterns == [
+        Pattern('/hmm'),
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
+        Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
     ]
 
 
 def test_dump_data_sources_with_missing_snapshot_errors():
     config = {'lvm': {}}
-    source_directories = ['/mnt/lvolume1/subdir', '/mnt/lvolume2']
+    patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
     flexmock(module).should_receive('get_logical_volumes').and_return(
         (
             module.Logical_volume(
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -510,7 +575,7 @@ def test_dump_data_sources_with_missing_snapshot_errors():
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
 
@@ -623,13 +688,13 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots():
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -669,6 +734,23 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots():
     )
 
 
+def test_remove_data_source_dumps_bails_for_missing_lvm_configuration():
+    flexmock(module).should_receive('get_logical_volumes').never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'replace_temporary_subdirectory_with_glob'
+    ).never()
+    flexmock(module).should_receive('unmount_snapshot').never()
+    flexmock(module).should_receive('remove_snapshot').never()
+
+    module.remove_data_source_dumps(
+        hook_config=None,
+        config={'source_directories': '/mnt/lvolume'},
+        log_prefix='test',
+        borgmatic_runtime_directory='/run/borgmatic',
+        dry_run=False,
+    )
+
+
 def test_remove_data_source_dumps_bails_for_missing_lsblk_command():
     config = {'lvm': {}}
     flexmock(module).should_receive('get_logical_volumes').and_raise(FileNotFoundError)
@@ -715,13 +797,13 @@ def test_remove_data_source_dumps_with_missing_snapshot_directory_skips_unmount(
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -760,13 +842,13 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -818,13 +900,13 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -876,13 +958,13 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command():
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -920,13 +1002,13 @@ def test_remove_data_source_dumps_bails_for_umount_command_error():
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -964,13 +1046,13 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command():
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -1008,13 +1090,13 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error():
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )
@@ -1054,13 +1136,13 @@ def test_remove_data_source_with_dry_run_skips_snapshot_unmount_and_delete():
                 name='lvolume1',
                 device_path='/dev/lvolume1',
                 mount_point='/mnt/lvolume1',
-                contained_source_directories=('/mnt/lvolume1/subdir',),
+                contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
             ),
             module.Logical_volume(
                 name='lvolume2',
                 device_path='/dev/lvolume2',
                 mount_point='/mnt/lvolume2',
-                contained_source_directories=('/mnt/lvolume2',),
+                contained_patterns=(Pattern('/mnt/lvolume2'),),
             ),
         )
     )

tests/unit/hooks/data_source/test_mariadb.py

@@ -80,7 +80,7 @@ def test_dump_data_sources_dumps_each_database():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -111,7 +111,7 @@ def test_dump_data_sources_dumps_with_password():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -137,7 +137,7 @@ def test_dump_data_sources_dumps_all_databases_at_once():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -166,7 +166,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -478,7 +478,7 @@ def test_dump_data_sources_errors_for_missing_all_databases():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
 
@@ -498,7 +498,7 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=True,
         )
         == []

tests/unit/hooks/data_source/test_mongodb.py

@@ -48,7 +48,7 @@ def test_dump_data_sources_runs_mongodump_for_each_database():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -71,7 +71,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=True,
         )
         == []
@@ -110,7 +110,7 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -156,7 +156,7 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -182,7 +182,7 @@ def test_dump_data_sources_runs_mongodump_with_directory_format():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == []
@@ -218,7 +218,7 @@ def test_dump_data_sources_runs_mongodump_with_options():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -244,7 +244,7 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 

tests/unit/hooks/data_source/test_mysql.py

@@ -80,7 +80,7 @@ def test_dump_data_sources_dumps_each_database():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -111,7 +111,7 @@ def test_dump_data_sources_dumps_with_password():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -137,7 +137,7 @@ def test_dump_data_sources_dumps_all_databases_at_once():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -166,7 +166,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -476,7 +476,7 @@ def test_dump_data_sources_errors_for_missing_all_databases():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
 
@@ -496,7 +496,7 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=True,
         )
         == []

tests/unit/hooks/data_source/test_postgresql.py

@@ -258,7 +258,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -278,7 +278,7 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
 
@@ -295,7 +295,7 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=True,
     ) == []
 
@@ -321,7 +321,7 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == []
@@ -349,7 +349,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=True,
         )
         == []
@@ -395,7 +395,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -439,7 +439,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -483,7 +483,7 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -523,7 +523,7 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == []
@@ -566,7 +566,7 @@ def test_dump_data_sources_runs_pg_dump_with_options():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -596,7 +596,7 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 
@@ -638,7 +638,7 @@ def test_dump_data_sources_runs_non_default_pg_dump():
         'test.yaml',
         config_paths=('test.yaml',),
         borgmatic_runtime_directory='/run/borgmatic',
-        source_directories=[],
+        patterns=[],
         dry_run=False,
     ) == [process]
 

tests/unit/hooks/data_source/test_snapshot.py

@@ -1,26 +1,34 @@
+from borgmatic.borg.pattern import Pattern
 from borgmatic.hooks.data_source import snapshot as module
 
 
-def test_get_contained_directories_without_candidates_returns_empty():
-    assert module.get_contained_directories('/mnt', {}) == ()
+def test_get_contained_patterns_without_candidates_returns_empty():
+    assert module.get_contained_patterns('/mnt', {}) == ()
 
 
-def test_get_contained_directories_with_self_candidate_returns_self():
-    candidates = {'/foo', '/mnt', '/bar'}
+def test_get_contained_patterns_with_self_candidate_returns_self():
+    candidates = {Pattern('/foo'), Pattern('/mnt'), Pattern('/bar')}
 
-    assert module.get_contained_directories('/mnt', candidates) == ('/mnt',)
-    assert candidates == {'/foo', '/bar'}
+    assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt'),)
+    assert candidates == {Pattern('/foo'), Pattern('/bar')}
 
 
-def test_get_contained_directories_with_child_candidate_returns_child():
-    candidates = {'/foo', '/mnt/subdir', '/bar'}
+def test_get_contained_patterns_with_self_candidate_and_caret_prefix_returns_self():
+    candidates = {Pattern('^/foo'), Pattern('^/mnt'), Pattern('^/bar')}
 
-    assert module.get_contained_directories('/mnt', candidates) == ('/mnt/subdir',)
-    assert candidates == {'/foo', '/bar'}
+    assert module.get_contained_patterns('/mnt', candidates) == (Pattern('^/mnt'),)
+    assert candidates == {Pattern('^/foo'), Pattern('^/bar')}
 
 
-def test_get_contained_directories_with_grandchild_candidate_returns_child():
-    candidates = {'/foo', '/mnt/sub/dir', '/bar'}
+def test_get_contained_patterns_with_child_candidate_returns_child():
+    candidates = {Pattern('/foo'), Pattern('/mnt/subdir'), Pattern('/bar')}
 
-    assert module.get_contained_directories('/mnt', candidates) == ('/mnt/sub/dir',)
-    assert candidates == {'/foo', '/bar'}
+    assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt/subdir'),)
+    assert candidates == {Pattern('/foo'), Pattern('/bar')}
+
+
+def test_get_contained_patterns_with_grandchild_candidate_returns_child():
+    candidates = {Pattern('/foo'), Pattern('/mnt/sub/dir'), Pattern('/bar')}
+
+    assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt/sub/dir'),)
+    assert candidates == {Pattern('/foo'), Pattern('/bar')}

tests/unit/hooks/data_source/test_sqlite.py

@@ -33,7 +33,7 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == []
@@ -64,7 +64,7 @@ def test_dump_data_sources_dumps_each_database():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -102,7 +102,7 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -131,7 +131,7 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -162,7 +162,7 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=False,
         )
         == processes
@@ -187,7 +187,7 @@ def test_dump_data_sources_does_not_dump_if_dry_run():
             'test.yaml',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=[],
+            patterns=[],
             dry_run=True,
         )
         == []

tests/unit/hooks/data_source/test_zfs.py

@@ -3,27 +3,35 @@ import os
 import pytest
 from flexmock import flexmock
 
+from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
 from borgmatic.hooks.data_source import zfs as module
 
 
-def test_get_datasets_to_backup_filters_datasets_by_source_directories():
+def test_get_datasets_to_backup_filters_datasets_by_patterns():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
     ).and_return(
         'dataset\t/dataset\t-\nother\t/other\t-',
     )
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
-    ).with_args('/dataset', object).and_return(('/dataset',))
+        'get_contained_patterns'
+    ).with_args('/dataset', object).and_return((Pattern('/dataset'),))
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).with_args('/other', object).and_return(())
 
     assert module.get_datasets_to_backup(
-        'zfs', source_directories=('/foo', '/dataset', '/bar')
+        'zfs',
+        patterns=(
+            Pattern('/foo'),
+            Pattern('/dataset'),
+            Pattern('/bar'),
+        ),
     ) == (
         module.Dataset(
-            name='dataset', mount_point='/dataset', contained_source_directories=('/dataset',)
+            name='dataset',
+            mount_point='/dataset',
+            contained_patterns=(Pattern('/dataset'),),
         ),
     )
 
@@ -35,18 +43,18 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property():
         'dataset\t/dataset\tauto\nother\t/other\t-',
     )
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
-    ).with_args('/dataset', object).never()
+        'get_contained_patterns'
+    ).with_args('/dataset', object).and_return(())
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).with_args('/other', object).and_return(())
 
-    assert module.get_datasets_to_backup('zfs', source_directories=('/foo', '/bar')) == (
+    assert module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar'))) == (
         module.Dataset(
             name='dataset',
             mount_point='/dataset',
             auto_backup=True,
-            contained_source_directories=('/dataset',),
+            contained_patterns=(Pattern('/dataset'),),
         ),
     )
 
@@ -58,11 +66,11 @@ def test_get_datasets_to_backup_with_invalid_list_output_raises():
         'dataset',
     )
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
+        'get_contained_patterns'
     ).never()
 
     with pytest.raises(ValueError, match='zfs'):
-        module.get_datasets_to_backup('zfs', source_directories=('/foo', '/bar'))
+        module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar')))
 
 
 def test_get_all_dataset_mount_points_does_not_filter_datasets():
@@ -72,8 +80,8 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets():
         '/dataset\n/other',
     )
     flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_directories'
-    ).and_return(('/dataset',))
+        'get_contained_patterns'
+    ).and_return((Pattern('/dataset'),))
 
     assert module.get_all_dataset_mount_points('zfs') == (
         ('/dataset'),
@@ -81,13 +89,47 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets():
     )
 
 
-def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories():
+@pytest.mark.parametrize(
+    'pattern,expected_pattern',
+    (
+        (
+            Pattern('/foo/bar/baz'),
+            Pattern('/run/borgmatic/zfs_snapshots/./foo/bar/baz'),
+        ),
+        (Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/./foo/bar')),
+        (
+            Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
+            Pattern(
+                '^/run/borgmatic/zfs_snapshots/./foo/bar',
+                Pattern_type.INCLUDE,
+                Pattern_style.REGULAR_EXPRESSION,
+            ),
+        ),
+        (
+            Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
+            Pattern(
+                '/run/borgmatic/zfs_snapshots/./foo/bar',
+                Pattern_type.INCLUDE,
+                Pattern_style.REGULAR_EXPRESSION,
+            ),
+        ),
+        (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/./foo')),
+        (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/./')),
+    ),
+)
+def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
+    pattern, expected_pattern
+):
+    assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern
+
+
+def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns():
     flexmock(module).should_receive('get_datasets_to_backup').and_return(
         (
             flexmock(
                 name='dataset',
                 mount_point='/mnt/dataset',
-                contained_source_directories=('/mnt/dataset/subdir',),
+                contained_patterns=(Pattern('/mnt/dataset/subdir'),),
             )
         )
     )
@@ -103,7 +145,10 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories()
         full_snapshot_name,
         module.os.path.normpath(snapshot_mount_path),
     ).once()
-    source_directories = ['/mnt/dataset/subdir']
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/dataset/subdir'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir'))
+    patterns = [Pattern('/mnt/dataset/subdir')]
 
     assert (
         module.dump_data_sources(
@@ -112,13 +157,13 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_source_directories()
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [os.path.join(snapshot_mount_path, 'subdir')]
+    assert patterns == [Pattern(os.path.join(snapshot_mount_path, 'subdir'))]
 
 
 def test_dump_data_sources_with_no_datasets_skips_snapshots():
@@ -126,22 +171,22 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots():
     flexmock(module.os).should_receive('getpid').and_return(1234)
     flexmock(module).should_receive('snapshot_dataset').never()
     flexmock(module).should_receive('mount_snapshot').never()
-    source_directories = ['/mnt/dataset']
+    patterns = [Pattern('/mnt/dataset')]
 
     assert (
         module.dump_data_sources(
             hook_config={},
-            config={'source_directories': '/mnt/dataset', 'zfs': {}},
+            config={'patterns': flexmock(), 'zfs': {}},
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == ['/mnt/dataset']
+    assert patterns == [Pattern('/mnt/dataset')]
 
 
 def test_dump_data_sources_uses_custom_commands():
@@ -150,7 +195,7 @@ def test_dump_data_sources_uses_custom_commands():
             flexmock(
                 name='dataset',
                 mount_point='/mnt/dataset',
-                contained_source_directories=('/mnt/dataset/subdir',),
+                contained_patterns=(Pattern('/mnt/dataset/subdir'),),
             )
         )
     )
@@ -166,7 +211,10 @@ def test_dump_data_sources_uses_custom_commands():
         full_snapshot_name,
         module.os.path.normpath(snapshot_mount_path),
     ).once()
-    source_directories = ['/mnt/dataset/subdir']
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/dataset/subdir'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir'))
+    patterns = [Pattern('/mnt/dataset/subdir')]
     hook_config = {
         'zfs_command': '/usr/local/bin/zfs',
         'mount_command': '/usr/local/bin/mount',
@@ -176,53 +224,53 @@ def test_dump_data_sources_uses_custom_commands():
         module.dump_data_sources(
             hook_config=hook_config,
             config={
-                'source_directories': source_directories,
+                'patterns': flexmock(),
                 'zfs': hook_config,
             },
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == [os.path.join(snapshot_mount_path, 'subdir')]
+    assert patterns == [Pattern(os.path.join(snapshot_mount_path, 'subdir'))]
 
 
-def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_source_directories():
+def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patterns():
     flexmock(module).should_receive('get_datasets_to_backup').and_return(
         (flexmock(name='dataset', mount_point='/mnt/dataset'),)
     )
     flexmock(module.os).should_receive('getpid').and_return(1234)
     flexmock(module).should_receive('snapshot_dataset').never()
     flexmock(module).should_receive('mount_snapshot').never()
-    source_directories = ['/mnt/dataset']
+    patterns = [Pattern('/mnt/dataset')]
 
     assert (
         module.dump_data_sources(
             hook_config={},
-            config={'source_directories': '/mnt/dataset', 'zfs': {}},
+            config={'patterns': ('R /mnt/dataset',), 'zfs': {}},
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=True,
         )
         == []
     )
 
-    assert source_directories == ['/mnt/dataset']
+    assert patterns == [Pattern('/mnt/dataset')]
 
 
-def test_dump_data_sources_ignores_mismatch_between_source_directories_and_contained_source_directories():
+def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns():
     flexmock(module).should_receive('get_datasets_to_backup').and_return(
         (
             flexmock(
                 name='dataset',
                 mount_point='/mnt/dataset',
-                contained_source_directories=('/mnt/dataset/subdir',),
+                contained_patterns=(Pattern('/mnt/dataset/subdir'),),
             )
         )
     )
@@ -238,22 +286,25 @@ def test_dump_data_sources_ignores_mismatch_between_source_directories_and_conta
         full_snapshot_name,
         module.os.path.normpath(snapshot_mount_path),
     ).once()
-    source_directories = ['/hmm']
+    flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
+        Pattern('/mnt/dataset/subdir'), '/run/borgmatic'
+    ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir'))
+    patterns = [Pattern('/hmm')]
 
     assert (
         module.dump_data_sources(
             hook_config={},
-            config={'source_directories': '/mnt/dataset', 'zfs': {}},
+            config={'patterns': ('R /mnt/dataset',), 'zfs': {}},
             log_prefix='test',
             config_paths=('test.yaml',),
             borgmatic_runtime_directory='/run/borgmatic',
-            source_directories=source_directories,
+            patterns=patterns,
             dry_run=False,
         )
         == []
     )
 
-    assert source_directories == ['/hmm', os.path.join(snapshot_mount_path, 'subdir')]
+    assert patterns == [Pattern('/hmm'), Pattern(os.path.join(snapshot_mount_path, 'subdir'))]
 
 
 def test_get_all_snapshots_parses_list_output():
@@ -321,6 +372,21 @@ def test_remove_data_source_dumps_use_custom_commands():
     )
 
 
+def test_remove_data_source_dumps_bails_for_missing_hook_configuration():
+    flexmock(module).should_receive('get_all_dataset_mount_points').never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'replace_temporary_subdirectory_with_glob'
+    ).never()
+
+    module.remove_data_source_dumps(
+        hook_config=None,
+        config={'source_directories': '/mnt/dataset'},
+        log_prefix='test',
+        borgmatic_runtime_directory='/run/borgmatic',
+        dry_run=False,
+    )
+
+
 def test_remove_data_source_dumps_bails_for_missing_zfs_command():
     flexmock(module).should_receive('get_all_dataset_mount_points').and_raise(FileNotFoundError)
     flexmock(module.borgmatic.config.paths).should_receive(

tests/unit/hooks/monitoring/test_sentry.py

@@ -0,0 +1,124 @@
+import pytest
+from flexmock import flexmock
+
+import borgmatic.hooks.monitoring.monitor
+from borgmatic.hooks.monitoring import sentry as module
+
+
+@pytest.mark.parametrize(
+    'state,configured_states,expected_status',
+    (
+        (borgmatic.hooks.monitoring.monitor.State.START, ['start'], 'in_progress'),
+        (
+            borgmatic.hooks.monitoring.monitor.State.START,
+            ['start', 'finish', 'fail'],
+            'in_progress',
+        ),
+        (borgmatic.hooks.monitoring.monitor.State.START, None, 'in_progress'),
+        (borgmatic.hooks.monitoring.monitor.State.FINISH, ['finish'], 'ok'),
+        (borgmatic.hooks.monitoring.monitor.State.FAIL, ['fail'], 'error'),
+    ),
+)
+def test_ping_monitor_constructs_cron_url_and_pings_it(state, configured_states, expected_status):
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+    }
+
+    if configured_states:
+        hook_config['states'] = configured_states
+
+    flexmock(module.requests).should_receive('get').with_args(
+        f'https://o294220.ingest.us.sentry.io/api/203069/cron/test/5f80ec/?status={expected_status}'
+    ).and_return(flexmock(ok=True))
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        state,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_unconfigured_state_bails():
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+        'states': ['fail'],
+    }
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+@pytest.mark.parametrize(
+    'data_source_name_url',
+    (
+        '5f80ec@o294220.ingest.us.sentry.io/203069',
+        'https://o294220.ingest.us.sentry.io/203069',
+        'https://5f80ec@/203069',
+        'https://5f80ec@o294220.ingest.us.sentry.io',
+        'https://5f80ec@o294220.ingest.us.sentry.io/203069/',
+    ),
+)
+def test_ping_monitor_with_invalid_data_source_name_url_bails(data_source_name_url):
+    hook_config = {
+        'data_source_name_url': data_source_name_url,
+        'monitor_slug': 'test',
+    }
+
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_invalid_sentry_state_bails():
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+        # This should never actually happen in practice, because the config schema prevents it.
+        'states': ['log'],
+    }
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.LOG,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_dry_run_bails():
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+    }
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=True,
+    )

tests/unit/hooks/test_dispatch.py

@@ -95,7 +95,7 @@ def test_call_hook_without_hook_config_invokes_module_function_with_arguments_an
         'borgmatic.hooks.monitoring.super_hook'
     ).and_return(test_module)
     flexmock(test_module).should_receive('hook_function').with_args(
-        {}, config, 'prefix', 55, value=66
+        None, config, 'prefix', 55, value=66
     ).and_return(expected_return_value).once()
 
     return_value = module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66)

tests/unit/test_logger.py

@@ -187,10 +187,14 @@ def test_console_color_formatter_format_includes_log_message():
 
 
 def test_color_text_does_not_raise():
-    module.color_text(module.colorama.Fore.RED, 'hi')
+    flexmock(module).should_receive('ansi_escape_code').and_return('blah')
+
+    module.color_text(module.Color.RED, 'hi')
 
 
 def test_color_text_without_color_does_not_raise():
+    flexmock(module).should_receive('ansi_escape_code').and_return('blah')
+
     module.color_text(None, 'hi')
 
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-env_list = py38,py39,py310,py311,py312
+env_list = py39,py310,py311,py312
 skip_missing_interpreters = True
 package = editable
 min_version = 4.0
@@ -28,10 +28,10 @@ commands =
 
 [testenv:end-to-end]
 package = editable
+system_site_packages = True
 deps =
     -r test_requirements.txt
-    pymongo==4.4.1
-       .
+    .
 pass_env = COVERAGE_FILE
 commands =
     pytest {posargs} --no-cov tests/end-to-end