diff --git a/NEWS b/NEWS index dd0bd93f..7e78180e 100644 --- a/NEWS +++ b/NEWS @@ -1,9 +1,15 @@ 1.9.11.dev0 - * #996: Fix the "create" action to omit the repository label prefix from Borg's output when - databases are enabled. * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ + * #996: Fix the "create" action to omit the repository label prefix from Borg's output when + databases are enabled. + * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from + a borgmatic configuration option (e.g. "source_directories")—not from other hooks within + borgmatic. + * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical + volumes. + * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property. * Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure than using an environment variable. * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index ab7815e4..dede448a 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -36,6 +36,7 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty path, borgmatic.borg.pattern.Pattern_type(pattern_type), borgmatic.borg.pattern.Pattern_style(pattern_style), + source=borgmatic.borg.pattern.Pattern_source.CONFIG, ) @@ -51,7 +52,9 @@ def collect_patterns(config): try: return ( tuple( - borgmatic.borg.pattern.Pattern(source_directory) + borgmatic.borg.pattern.Pattern( + source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG + ) for source_directory in config.get('source_directories', ()) ) + tuple( @@ -144,6 +147,7 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None): pattern.type, pattern.style, pattern.device, + pattern.source, ) for expanded_path in expand_directory(pattern.path, working_directory) ) @@ -178,6 +182,7 @@ def device_map_patterns(patterns, working_directory=None): and os.path.exists(full_path) else None ), + source=pattern.source, ) for pattern in patterns for full_path in (os.path.join(working_directory or '', pattern.path),) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 9436afd4..aea0cbc8 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -132,9 +132,12 @@ def collect_special_file_paths( used. Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates - its own special files there for database dumps. And if the borgmatic runtime directory is - configured to be excluded from the files Borg backs up, error, because this means Borg won't be - able to consume any database dumps and therefore borgmatic will hang. + its own special files there for database dumps and we don't want those omitted. + + Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg + plans to backup, that means the user must have excluded the runtime directory (e.g. via + "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to + consume any database dumps and therefore borgmatic will hang when it tries to do so. ''' # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open # files including any named pipe we've created. @@ -148,25 +151,33 @@ def collect_special_file_paths( borg_exit_codes=config.get('borg_exit_codes'), ) + # These are all the individual files that Borg is planning to backup as determined by the Borg + # create dry run above. paths = tuple( path_line.split(' ', 1)[1] for path_line in paths_output.split('\n') if path_line and path_line.startswith('- ') or path_line.startswith('+ ') ) - skip_paths = {} + + # These are the subset of those files that contain the borgmatic runtime directory. + paths_containing_runtime_directory = {} if os.path.exists(borgmatic_runtime_directory): - skip_paths = { + paths_containing_runtime_directory = { path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,)) } - if not skip_paths and not dry_run: + # If no paths to backup contain the runtime directory, it must've been excluded. + if not paths_containing_runtime_directory and not dry_run: raise ValueError( f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.' ) return tuple( - path for path in paths if special_file(path, working_directory) if path not in skip_paths + path + for path in paths + if special_file(path, working_directory) + if path not in paths_containing_runtime_directory ) @@ -325,6 +336,7 @@ def make_base_create_command( special_file_path, borgmatic.borg.pattern.Pattern_type.NO_RECURSE, borgmatic.borg.pattern.Pattern_style.FNMATCH, + source=borgmatic.borg.pattern.Pattern_source.INTERNAL, ) for special_file_path in special_file_paths ), diff --git a/borgmatic/borg/pattern.py b/borgmatic/borg/pattern.py index be57bc0e..dfce7c3b 100644 --- a/borgmatic/borg/pattern.py +++ b/borgmatic/borg/pattern.py @@ -20,12 +20,31 @@ class Pattern_style(enum.Enum): PATH_FULL_MATCH = 'pf' +class Pattern_source(enum.Enum): + ''' + Where the pattern came from within borgmatic. This is important because certain use cases (like + filesystem snapshotting) only want to consider patterns that the user actually put in a + configuration file and not patterns from other sources. + ''' + + # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories". + CONFIG = 'config' + + # The pattern is generated internally within borgmatic, e.g. for special file excludes. + INTERNAL = 'internal' + + # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump + # directory. + HOOK = 'hook' + + Pattern = collections.namedtuple( 'Pattern', - ('path', 'type', 'style', 'device'), + ('path', 'type', 'style', 'device', 'source'), defaults=( Pattern_type.ROOT, Pattern_style.NONE, None, + Pattern_source.HOOK, ), ) diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index 67d75421..fe779f0e 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -55,9 +55,17 @@ def dump_data_sources( manifest_file, ) - patterns.extend(borgmatic.borg.pattern.Pattern(config_path) for config_path in config_paths) + patterns.extend( + borgmatic.borg.pattern.Pattern( + config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK + ) + for config_path in config_paths + ) patterns.append( - borgmatic.borg.pattern.Pattern(os.path.join(borgmatic_runtime_directory, 'bootstrap')) + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'bootstrap'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) ) return [] diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 2971aed8..ec6d33df 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -54,7 +54,9 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None): between the current Btrfs filesystem and subvolume mount points and the paths of any patterns. The idea is that these pattern paths represent the requested subvolumes to snapshot. - If patterns is None, then return all subvolumes, sorted by path. + Only include subvolumes that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then + return all subvolumes instead, sorted by path. Return the result as a sequence of matching subvolume mount points. ''' @@ -73,7 +75,12 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None): mount_point, candidate_patterns ), ) - if patterns is None or contained_patterns + if patterns is None + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ) return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path)) @@ -121,6 +128,7 @@ def make_snapshot_exclude_pattern(subvolume_path): # pragma: no cover ), borgmatic.borg.pattern.Pattern_type.NO_RECURSE, borgmatic.borg.pattern.Pattern_style.FNMATCH, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -153,6 +161,7 @@ def make_borg_snapshot_pattern(subvolume_path, pattern): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -198,7 +207,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') - # Based on the configured patterns, determine Btrfs subvolumes to backup. + # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those + # patterns that came from actual user configuration (as opposed to, say, other hooks). btrfs_command = hook_config.get('btrfs_command', 'btrfs') findmnt_command = hook_config.get('findmnt_command', 'findmnt') subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 8436d0f6..11cd1c09 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -1,5 +1,6 @@ import collections import glob +import hashlib import json import logging import os @@ -33,7 +34,9 @@ def get_logical_volumes(lsblk_command, patterns=None): between the current LVM logical volume mount points and the paths of any patterns. The idea is that these pattern paths represent the requested logical volumes to snapshot. - If patterns is None, include all logical volume mounts points, not just those in patterns. + Only include logical volumes that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, include + all logical volume mounts points instead, not just those in patterns. Return the result as a sequence of Logical_volume instances. ''' @@ -72,7 +75,12 @@ def get_logical_volumes(lsblk_command, patterns=None): device['mountpoint'], candidate_patterns ), ) - if not patterns or contained_patterns + if not patterns + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ) except KeyError as error: raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"') @@ -124,10 +132,14 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path): # prag ) -def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): +MOUNT_POINT_HASH_LENGTH = 10 + + +def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory): ''' - Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its - path rewritten to be in a snapshot directory based on the given runtime directory. + Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume + containing it, return a new Pattern with its path rewritten to be in a snapshot directory based + on both the given runtime directory and the given Logical_volume's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break the regular expression. @@ -142,6 +154,13 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): rewritten_path = initial_caret + os.path.join( normalized_runtime_directory, 'lvm_snapshots', + # Including this hash prevents conflicts between snapshot patterns for different logical + # volumes. For instance, without this, snapshotting a logical volume at /var and another at + # /var/spool would result in overlapping snapshot patterns and therefore colliding mount + # attempts. + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -152,6 +171,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -180,7 +200,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') - # List logical volumes to get their mount points. + # List logical volumes to get their mount points, but only consider those patterns that came + # from actual user configuration (as opposed to, say, other hooks). lsblk_command = hook_config.get('lsblk_command', 'lsblk') requested_logical_volumes = get_logical_volumes(lsblk_command, patterns) @@ -218,6 +239,9 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'lvm_snapshots', + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), logical_volume.mount_point.lstrip(os.path.sep), ) @@ -233,7 +257,9 @@ def dump_data_sources( ) for pattern in logical_volume.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory) + snapshot_pattern = make_borg_snapshot_pattern( + pattern, logical_volume, normalized_runtime_directory + ) # Attempt to update the pattern in place, since pattern order matters to Borg. try: @@ -337,6 +363,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d os.path.normpath(borgmatic_runtime_directory), ), 'lvm_snapshots', + '*', ) logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') @@ -349,7 +376,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d snapshot_mount_path = os.path.join( snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep) ) - if not os.path.isdir(snapshot_mount_path): + + # If the snapshot mount path is empty, this is probably just a "shadow" of a nested + # logical volume and therefore there's nothing to unmount. + if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path): continue # This might fail if the directory is already mounted, but we swallow errors here since @@ -374,7 +404,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d return except subprocess.CalledProcessError as error: logger.debug(error) - return + continue if not dry_run: shutil.rmtree(snapshots_directory) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 727932ee..b992a944 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -216,7 +216,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mariadb_databases') + os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 2148f009..59811101 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -81,7 +81,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mongodb_databases') + os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index ffa2bc3f..3172edbb 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -215,7 +215,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mysql_databases') + os.path.join(borgmatic_runtime_directory, 'mysql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 8cf17518..559cc4b8 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -239,7 +239,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'postgresql_databases') + os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/snapshot.py b/borgmatic/hooks/data_source/snapshot.py index 2ee08a8f..ece63ce6 100644 --- a/borgmatic/hooks/data_source/snapshot.py +++ b/borgmatic/hooks/data_source/snapshot.py @@ -11,10 +11,10 @@ def get_contained_patterns(parent_directory, candidate_patterns): paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then /var is what we want to snapshot. - For this to work, a candidate pattern path can't have any globs or other non-literal characters - in the initial portion of the path that matches the parent directory. For instance, a parent - directory of /var would match a candidate pattern path of /var/log/*/data, but not a pattern - path like /v*/log/*/data. + For this function to work, a candidate pattern path can't have any globs or other non-literal + characters in the initial portion of the path that matches the parent directory. For instance, a + parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a + pattern path like /v*/log/*/data. The one exception is that if a regular expression pattern path starts with "^", that will get stripped off for purposes of matching against a parent directory. @@ -31,8 +31,10 @@ def get_contained_patterns(parent_directory, candidate_patterns): candidate for candidate in candidate_patterns for candidate_path in (pathlib.PurePath(candidate.path.lstrip('^')),) - if pathlib.PurePath(parent_directory) == candidate_path - or pathlib.PurePath(parent_directory) in candidate_path.parents + if ( + pathlib.PurePath(parent_directory) == candidate_path + or pathlib.PurePath(parent_directory) in candidate_path.parents + ) ) candidate_patterns -= set(contained_patterns) diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 4d5ce9a0..b36c0ccb 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -90,7 +90,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'sqlite_databases') + os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 82a71e7a..a7e9f01d 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -1,5 +1,6 @@ import collections import glob +import hashlib import logging import os import shutil @@ -38,6 +39,9 @@ def get_datasets_to_backup(zfs_command, patterns): pattern paths represent the requested datasets to snapshot. But also include any datasets tagged with a borgmatic-specific user property, whether or not they appear in the patterns. + Only include datasets that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). + Return the result as a sequence of Dataset instances, sorted by mount point. ''' list_output = borgmatic.execute.execute_command_and_capture_output( @@ -48,7 +52,7 @@ def get_datasets_to_backup(zfs_command, patterns): '-t', 'filesystem', '-o', - f'name,mountpoint,{BORGMATIC_USER_PROPERTY}', + f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}', ) ) @@ -60,7 +64,12 @@ def get_datasets_to_backup(zfs_command, patterns): ( Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ()) for line in list_output.splitlines() - for (dataset_name, mount_point, user_property_value) in (line.rstrip().split('\t'),) + for (dataset_name, mount_point, can_mount, user_property_value) in ( + line.rstrip().split('\t'), + ) + # Skip datasets that are marked "canmount=off", because mounting their snapshots will + # result in completely empty mount points—thereby preventing us from backing them up. + if can_mount == 'on' ), key=lambda dataset: dataset.mount_point, reverse=True, @@ -83,7 +92,12 @@ def get_datasets_to_backup(zfs_command, patterns): for contained_patterns in ( ( ( - (borgmatic.borg.pattern.Pattern(dataset.mount_point),) + ( + borgmatic.borg.pattern.Pattern( + dataset.mount_point, + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ), + ) if dataset.auto_backup else () ) @@ -92,7 +106,12 @@ def get_datasets_to_backup(zfs_command, patterns): ) ), ) - if contained_patterns + if dataset.auto_backup + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ), key=lambda dataset: dataset.mount_point, ) @@ -155,10 +174,14 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p ) -def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): +MOUNT_POINT_HASH_LENGTH = 10 + + +def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory): ''' - Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its - path rewritten to be in a snapshot directory based on the given runtime directory. + Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and the Dataset containing it, + return a new Pattern with its path rewritten to be in a snapshot directory based on both the + given runtime directory and the given Dataset's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break the regular expression. @@ -173,6 +196,10 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): rewritten_path = initial_caret + os.path.join( normalized_runtime_directory, 'zfs_snapshots', + # Including this hash prevents conflicts between snapshot patterns for different datasets. + # For instance, without this, snapshotting a dataset at /var and another at /var/spool would + # result in overlapping snapshot patterns and therefore colliding mount attempts. + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -183,6 +210,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -209,7 +237,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting ZFS datasets{dry_run_label}') - # List ZFS datasets to get their mount points. + # List ZFS datasets to get their mount points, but only consider those patterns that came from + # actual user configuration (as opposed to, say, other hooks). zfs_command = hook_config.get('zfs_command', 'zfs') requested_datasets = get_datasets_to_backup(zfs_command, patterns) @@ -234,6 +263,9 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'zfs_snapshots', + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), dataset.mount_point.lstrip(os.path.sep), ) @@ -249,7 +281,9 @@ def dump_data_sources( ) for pattern in dataset.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory) + snapshot_pattern = make_borg_snapshot_pattern( + pattern, dataset, normalized_runtime_directory + ) # Attempt to update the pattern in place, since pattern order matters to Borg. try: @@ -334,6 +368,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d os.path.normpath(borgmatic_runtime_directory), ), 'zfs_snapshots', + '*', ) logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') @@ -346,7 +381,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d # child datasets before the shorter mount point paths of parent datasets. for mount_point in reversed(dataset_mount_points): snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep)) - if not os.path.isdir(snapshot_mount_path): + + # If the snapshot mount path is empty, this is probably just a "shadow" of a nested + # dataset and therefore there's nothing to unmount. + if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path): continue # This might fail if the path is already mounted, but we swallow errors here since we'll @@ -370,7 +408,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d return except subprocess.CalledProcessError as error: logger.debug(error) - return + continue if not dry_run: shutil.rmtree(snapshots_directory) diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index a9f2d267..f1b6f8e2 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -54,8 +54,8 @@ You have a couple of options for borgmatic to find and backup your ZFS datasets: * For any dataset you'd like backed up, add its mount point to borgmatic's `source_directories` option. * New in version 1.9.6 Or - include the mount point with borgmatic's `patterns` or `patterns_from` - options. + include the mount point as a root pattern with borgmatic's `patterns` or + `patterns_from` options. * Or set the borgmatic-specific user property `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find @@ -65,6 +65,11 @@ If you have multiple borgmatic configuration files with ZFS enabled, and you'd like particular datasets to be backed up only for particular configuration files, use the `source_directories` option instead of the user property. +New in version 1.9.11 borgmatic +won't snapshot datasets with the `canmount=off` property, which is often set on +datasets that only serve as a container for other datasets. Use `zfs get +canmount datasetname` to see the `canmount` value for a dataset. + During a backup, borgmatic automatically snapshots these discovered datasets (non-recursively), temporarily mounts the snapshots within its [runtime directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory), @@ -147,7 +152,8 @@ For any subvolume you'd like backed up, add its path to borgmatic's `source_directories` option. New in version 1.9.6 Or include -the mount point with borgmatic's `patterns` or `patterns_from` options. +the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` +options. During a backup, borgmatic snapshots these subvolumes (non-recursively) and includes the snapshotted files in the paths sent to Borg. borgmatic is also @@ -252,7 +258,8 @@ For any logical volume you'd like backed up, add its mount point to borgmatic's `source_directories` option. New in version 1.9.6 Or include -the mount point with borgmatic's `patterns` or `patterns_from` options. +the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` +options. During a backup, borgmatic automatically snapshots these discovered logical volumes (non-recursively), temporarily mounts the snapshots within its [runtime diff --git a/tests/end-to-end/commands/fake_zfs.py b/tests/end-to-end/commands/fake_zfs.py index 61f8ae5e..3c13f4d5 100644 --- a/tests/end-to-end/commands/fake_zfs.py +++ b/tests/end-to-end/commands/fake_zfs.py @@ -27,6 +27,7 @@ BUILTIN_DATASETS = ( 'used': '256K', 'avail': '23.7M', 'refer': '25K', + 'canmount': 'on', 'mountpoint': '/pool', }, { @@ -34,6 +35,7 @@ BUILTIN_DATASETS = ( 'used': '256K', 'avail': '23.7M', 'refer': '25K', + 'canmount': 'on', 'mountpoint': '/pool/dataset', }, ) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 8ba1eed0..5e4a4c9f 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -5,16 +5,21 @@ import pytest from flexmock import flexmock from borgmatic.actions import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type @pytest.mark.parametrize( 'pattern_line,expected_pattern', ( - ('R /foo', Pattern('/foo')), - ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)), - ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)), - ('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)), + ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)), + ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)), + ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)), + ( + '+ sh:/foo*', + Pattern( + '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), + ), ), ) def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern): @@ -28,8 +33,8 @@ def test_parse_pattern_with_invalid_pattern_line_errors(): def test_collect_patterns_converts_source_directories(): assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == ( - Pattern('/foo'), - Pattern('/bar'), + Pattern('/foo', source=Pattern_source.CONFIG), + Pattern('/bar', source=Pattern_source.CONFIG), ) @@ -48,9 +53,15 @@ def test_collect_patterns_parses_config_patterns(): def test_collect_patterns_converts_exclude_patterns(): assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == ( - Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL), + Pattern( + '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 94e02425..3f1f09a7 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -4,7 +4,7 @@ import pytest from flexmock import flexmock from borgmatic.borg import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from ..test_verbosity import insert_logging_mock @@ -663,6 +663,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', @@ -713,6 +714,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index dde7b05b..648902c3 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import btrfs as module @@ -52,9 +52,14 @@ def test_get_subvolume_mount_points_with_findmnt_json_missing_filesystems_errors def test_get_subvolumes_collects_subvolumes_matching_patterns(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + contained_pattern = Pattern( + '/mnt1', + type=Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' - ).with_args('/mnt1', object).and_return((Pattern('/mnt1'),)) + ).with_args('/mnt1', object).and_return((contained_pattern,)) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/mnt2', object).and_return(()) @@ -66,7 +71,69 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns(): Pattern('/mnt1'), Pattern('/mnt3'), ], - ) == (module.Subvolume('/mnt1', contained_patterns=(Pattern('/mnt1'),)),) + ) == (module.Subvolume('/mnt1', contained_patterns=(contained_pattern,)),) + + +def test_get_subvolumes_skips_non_root_patterns(): + flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt1', object).and_return( + ( + Pattern( + '/mnt1', + type=Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt2', object).and_return(()) + + assert ( + module.get_subvolumes( + 'btrfs', + 'findmnt', + patterns=[ + Pattern('/mnt1'), + Pattern('/mnt3'), + ], + ) + == () + ) + + +def test_get_subvolumes_skips_non_config_patterns(): + flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt1', object).and_return( + ( + Pattern( + '/mnt1', + type=Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt2', object).and_return(()) + + assert ( + module.get_subvolumes( + 'btrfs', + 'findmnt', + patterns=[ + Pattern('/mnt1'), + Pattern('/mnt3'), + ], + ) + == () + ) def test_get_subvolumes_without_patterns_collects_all_subvolumes(): diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 49159905..34cf5250 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import lvm as module @@ -37,14 +37,20 @@ def test_get_logical_volumes_filters_by_patterns(): } ''' ) - contained = {Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')} + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + } flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args(None, contained).never() flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/mnt/lvolume', contained).and_return( - (Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + ( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ) ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -54,17 +60,116 @@ def test_get_logical_volumes_filters_by_patterns(): ).with_args('/mnt/notlvm', contained).never() assert module.get_logical_volumes( - 'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ) == ( module.Logical_volume( name='vgroup-lvolume', device_path='/dev/mapper/vgroup-lvolume', mount_point='/mnt/lvolume', - contained_patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')), + contained_patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ), ) +def test_get_logical_volumes_skips_non_root_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern( + '/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG + ), + ), + ) + == () + ) + + +def test_get_logical_volumes_skips_non_config_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ), + ) + == () + ) + + def test_get_logical_volumes_with_invalid_lsblk_json_errors(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -138,13 +243,13 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( ( Pattern('/foo/bar/baz'), - Pattern('/run/borgmatic/lvm_snapshots/./foo/bar/baz'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar/baz'), ), - (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/./foo/bar')), + (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar')), ( Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '^/run/borgmatic/lvm_snapshots/./foo/bar', + '^/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), @@ -152,40 +257,48 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '/run/borgmatic/lvm_snapshots/./foo/bar', + '/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), ), - (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/./foo')), - (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/./')), + (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')), + (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( pattern, expected_pattern ): - assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + + assert ( + module.make_borg_snapshot_pattern( + pattern, flexmock(mount_point='/something'), '/run/borgmatic' + ) + == expected_pattern + ) def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -203,18 +316,21 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -229,8 +345,8 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -259,22 +375,21 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): def test_dump_data_sources_uses_snapshot_size_for_snapshot(): config = {'lvm': {'snapshot_size': '1000PB'}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', @@ -298,18 +413,21 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -324,8 +442,8 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -339,22 +457,21 @@ def test_dump_data_sources_uses_custom_commands(): }, } patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( '/usr/local/bin/lvcreate', @@ -378,18 +495,25 @@ def test_dump_data_sources_uses_custom_commands(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/usr/local/bin/mount', + '/dev/lvolume1_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/usr/local/bin/mount', + '/dev/lvolume2_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -404,8 +528,8 @@ def test_dump_data_sources_uses_custom_commands(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -463,22 +587,21 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): config = {'lvm': {}} patterns = [Pattern('/hmm')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -496,18 +619,21 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -523,8 +649,8 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained assert patterns == [ Pattern('/hmm'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -694,16 +820,19 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -799,9 +928,11 @@ def test_remove_data_source_dumps_with_missing_snapshot_directory_skips_unmount( flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(False) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() @@ -843,24 +974,92 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', + ).once() + flexmock(module).should_receive('get_snapshots').and_return( + ( + module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'), + module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'), + ), + ) + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once() + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once() + + module.remove_data_source_dumps( + hook_config=config['lvm'], + config=config, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_empty_snapshot_mount_path_skips_unmount(): + config = {'lvm': {}} + flexmock(module).should_receive('get_logical_volumes').and_return( + ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), + ) + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).and_return('/run/borgmatic') + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' + ).and_return([]) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' + ).and_return(['file.txt']) + flexmock(module.shutil).should_receive('rmtree') + flexmock(module).should_receive('unmount_snapshot').with_args( + 'umount', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', + ).never() + flexmock(module).should_receive('unmount_snapshot').with_args( + 'umount', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -900,24 +1099,27 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(True).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True).and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -957,16 +1159,19 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(FileNotFoundError) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).never() flexmock(module).should_receive('get_snapshots').never() flexmock(module).should_receive('remove_snapshot').never() @@ -979,7 +1184,7 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): ) -def test_remove_data_source_dumps_bails_for_umount_command_error(): +def test_remove_data_source_dumps_swallows_umount_command_error(): config = {'lvm': {}} flexmock(module).should_receive('get_logical_volumes').and_return( ( @@ -1000,19 +1205,28 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(module.subprocess.CalledProcessError(1, 'wtf')) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', - ).never() - flexmock(module).should_receive('get_snapshots').never() - flexmock(module).should_receive('remove_snapshot').never() + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', + ).once() + flexmock(module).should_receive('get_snapshots').and_return( + ( + module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'), + module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'), + ), + ) + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once() + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once() module.remove_data_source_dumps( hook_config=config['lvm'], @@ -1043,16 +1257,19 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise(FileNotFoundError) flexmock(module).should_receive('remove_snapshot').never() @@ -1086,16 +1303,19 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise( module.subprocess.CalledProcessError(1, 'wtf') @@ -1133,6 +1353,7 @@ def test_remove_data_source_with_dry_run_skips_snapshot_unmount_and_delete(): ).and_return('/run/borgmatic') flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_snapshots').and_return( diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 85782bdb..fbe18fa9 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -3,7 +3,7 @@ import os import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import zfs as module @@ -11,11 +11,19 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - 'dataset\t/dataset\t-\nother\t/other\t-', + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' - ).with_args('/dataset', object).and_return((Pattern('/dataset'),)) + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ) + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/other', object).and_return(()) @@ -23,24 +31,134 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): assert module.get_datasets_to_backup( 'zfs', patterns=( - Pattern('/foo'), - Pattern('/dataset'), - Pattern('/bar'), + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), ), ) == ( module.Dataset( name='dataset', mount_point='/dataset', - contained_patterns=(Pattern('/dataset'),), + contained_patterns=( + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ), ), ) +def test_get_datasets_to_backup_skips_non_root_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ), + ) + == () + ) + + +def test_get_datasets_to_backup_skips_non_config_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ), + ) + == () + ) + + def test_get_datasets_to_backup_filters_datasets_by_user_property(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - 'dataset\t/dataset\tauto\nother\t/other\t-', + 'dataset\t/dataset\ton\tauto\nother\t/other\ton\t-', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -49,16 +167,45 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property(): 'get_contained_patterns' ).with_args('/other', object).and_return(()) - assert module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar'))) == ( + assert module.get_datasets_to_backup( + 'zfs', + patterns=(Pattern('/foo'), Pattern('/bar')), + ) == ( module.Dataset( name='dataset', mount_point='/dataset', auto_backup=True, - contained_patterns=(Pattern('/dataset'),), + contained_patterns=(Pattern('/dataset', source=Pattern_source.HOOK),), ), ) +def test_get_datasets_to_backup_filters_datasets_by_canmount_property(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\toff\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return((Pattern('/dataset'),)) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern('/foo'), + Pattern('/dataset'), + Pattern('/bar'), + ), + ) + == () + ) + + def test_get_datasets_to_backup_with_invalid_list_output_raises(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -94,13 +241,13 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets(): ( ( Pattern('/foo/bar/baz'), - Pattern('/run/borgmatic/zfs_snapshots/./foo/bar/baz'), + Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar/baz'), ), - (Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/./foo/bar')), + (Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar')), ( Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '^/run/borgmatic/zfs_snapshots/./foo/bar', + '^/run/borgmatic/zfs_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), @@ -108,46 +255,55 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets(): ( Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '/run/borgmatic/zfs_snapshots/./foo/bar', + '/run/borgmatic/zfs_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), ), - (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/./foo')), - (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/./')), + (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')), + (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( pattern, expected_pattern ): - assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + + assert ( + module.make_borg_snapshot_pattern( + pattern, flexmock(mount_point='/something'), '/run/borgmatic' + ) + == expected_pattern + ) def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( 'zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( 'mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/mnt/dataset/subdir')] assert ( @@ -188,30 +344,30 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots(): def test_dump_data_sources_uses_custom_commands(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( '/usr/local/bin/zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( '/usr/local/bin/mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/mnt/dataset/subdir')] hook_config = { 'zfs_command': '/usr/local/bin/zfs', @@ -261,30 +417,30 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patter def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( 'zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( 'mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/hmm')] assert ( @@ -317,11 +473,14 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - 'umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + 'umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).once() flexmock(module).should_receive('get_all_snapshots').and_return( ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') @@ -343,11 +502,14 @@ def test_remove_data_source_dumps_use_custom_commands(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).once() flexmock(module).should_receive('get_all_snapshots').and_return( ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') @@ -416,11 +578,14 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_raise(FileNotFoundError) flexmock(module).should_receive('get_all_snapshots').never() flexmock(module).should_receive('destroy_snapshot').never() @@ -434,19 +599,26 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): ) -def test_remove_data_source_dumps_bails_for_umount_command_error(): +def test_remove_data_source_dumps_swallows_umount_command_error(): flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',)) flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_raise(module.subprocess.CalledProcessError(1, 'wtf')) - flexmock(module).should_receive('get_all_snapshots').never() - flexmock(module).should_receive('destroy_snapshot').never() + flexmock(module).should_receive('get_all_snapshots').and_return( + ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') + ) + flexmock(module).should_receive('destroy_snapshot').with_args( + '/usr/local/bin/zfs', 'dataset@borgmatic-1234' + ).once() hook_config = {'zfs_command': '/usr/local/bin/zfs', 'umount_command': '/usr/local/bin/umount'} module.remove_data_source_dumps( @@ -462,7 +634,9 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_no flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(False) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() @@ -486,13 +660,50 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots' + '/run/borgmatic/zfs_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) + flexmock(module.shutil).should_receive('rmtree') + flexmock(module).should_receive('unmount_snapshot').never() + flexmock(module).should_receive('get_all_snapshots').and_return( + ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') + ) + flexmock(module).should_receive('destroy_snapshot').with_args( + 'zfs', 'dataset@borgmatic-1234' + ).once() + + module.remove_data_source_dumps( + hook_config={}, + config={'source_directories': '/mnt/dataset', 'zfs': {}}, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_empty(): + flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',)) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).and_return('/run/borgmatic') + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' + ).and_return([]) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( @@ -515,13 +726,16 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots' + '/run/borgmatic/zfs_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(True).and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( @@ -544,8 +758,11 @@ def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return(