peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-22 02:03:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into passphrase-via-file-descriptor

Browse Source
This commit is contained in:
Dan Helfman
2025-02-21 20:26:48 -08:00
parent 07ecc0ffd6 cc2ede70ac
commit 4ee2603fef
21 changed files with 933 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS
+8 -2
View File
@@ -1,9 +1,15 @@
1.9.11.dev0
* #996: Fix the "create" action to omit the repository label prefix from Borg's output when
databases are enabled.
* #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the
documentation for more information:
https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/
* #996: Fix the "create" action to omit the repository label prefix from Borg's output when
databases are enabled.
* #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from
a borgmatic configuration option (e.g. "source_directories")—not from other hooks within
borgmatic.
* #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical
volumes.
* #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property.
* Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure than
using an environment variable.
* Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
borgmatic/actions/create.py
+6 -1
View File
@@ -36,6 +36,7 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty
path,
borgmatic.borg.pattern.Pattern_type(pattern_type),
borgmatic.borg.pattern.Pattern_style(pattern_style),
source=borgmatic.borg.pattern.Pattern_source.CONFIG,
)
@@ -51,7 +52,9 @@ def collect_patterns(config):
try:
return (
tuple(
borgmatic.borg.pattern.Pattern(source_directory)
borgmatic.borg.pattern.Pattern(
source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
)
for source_directory in config.get('source_directories', ())
)
+ tuple(
@@ -144,6 +147,7 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None):
pattern.type,
pattern.style,
pattern.device,
pattern.source,
)
for expanded_path in expand_directory(pattern.path, working_directory)
)
@@ -178,6 +182,7 @@ def device_map_patterns(patterns, working_directory=None):
and os.path.exists(full_path)
else None
),
source=pattern.source,
)
for pattern in patterns
for full_path in (os.path.join(working_directory or '', pattern.path),)
borgmatic/borg/create.py
+19 -7
View File
@@ -132,9 +132,12 @@ def collect_special_file_paths(
used.
Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
its own special files there for database dumps. And if the borgmatic runtime directory is
configured to be excluded from the files Borg backs up, error, because this means Borg won't be
able to consume any database dumps and therefore borgmatic will hang.
its own special files there for database dumps and we don't want those omitted.
Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
plans to backup, that means the user must have excluded the runtime directory (e.g. via
"exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
consume any database dumps and therefore borgmatic will hang when it tries to do so.
'''
# Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
# files including any named pipe we've created.
@@ -148,25 +151,33 @@ def collect_special_file_paths(
borg_exit_codes=config.get('borg_exit_codes'),
)
# These are all the individual files that Borg is planning to backup as determined by the Borg
# create dry run above.
paths = tuple(
path_line.split(' ', 1)[1]
for path_line in paths_output.split('\n')
if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
)
skip_paths = {}
# These are the subset of those files that contain the borgmatic runtime directory.
paths_containing_runtime_directory = {}
if os.path.exists(borgmatic_runtime_directory):
skip_paths = {
paths_containing_runtime_directory = {
path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
}
if not skip_paths and not dry_run:
# If no paths to backup contain the runtime directory, it must've been excluded.
if not paths_containing_runtime_directory and not dry_run:
raise ValueError(
f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
)
return tuple(
path for path in paths if special_file(path, working_directory) if path not in skip_paths
path
for path in paths
if special_file(path, working_directory)
if path not in paths_containing_runtime_directory
)
@@ -325,6 +336,7 @@ def make_base_create_command(
special_file_path,
borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
borgmatic.borg.pattern.Pattern_style.FNMATCH,
source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
)
for special_file_path in special_file_paths
),
borgmatic/borg/pattern.py
+20 -1
View File
@@ -20,12 +20,31 @@ class Pattern_style(enum.Enum):
PATH_FULL_MATCH = 'pf'
class Pattern_source(enum.Enum):
'''
Where the pattern came from within borgmatic. This is important because certain use cases (like
filesystem snapshotting) only want to consider patterns that the user actually put in a
configuration file and not patterns from other sources.
'''
# The pattern is from a borgmatic configuration option, e.g. listed in "source_directories".
CONFIG = 'config'
# The pattern is generated internally within borgmatic, e.g. for special file excludes.
INTERNAL = 'internal'
# The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump
# directory.
HOOK = 'hook'
Pattern = collections.namedtuple(
'Pattern',
('path', 'type', 'style', 'device'),
('path', 'type', 'style', 'device', 'source'),
defaults=(
Pattern_type.ROOT,
Pattern_style.NONE,
None,
Pattern_source.HOOK,
),
)
borgmatic/hooks/data_source/bootstrap.py
+10 -2
View File
@@ -55,9 +55,17 @@ def dump_data_sources(
manifest_file,
)
patterns.extend(borgmatic.borg.pattern.Pattern(config_path) for config_path in config_paths)
patterns.extend(
borgmatic.borg.pattern.Pattern(
config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK
)
for config_path in config_paths
)
patterns.append(
borgmatic.borg.pattern.Pattern(os.path.join(borgmatic_runtime_directory, 'bootstrap'))
borgmatic.borg.pattern.Pattern(
os.path.join(borgmatic_runtime_directory, 'bootstrap'),
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
)
return []
borgmatic/hooks/data_source/btrfs.py
+13 -3
View File
@@ -54,7 +54,9 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
between the current Btrfs filesystem and subvolume mount points and the paths of any patterns.
The idea is that these pattern paths represent the requested subvolumes to snapshot.
If patterns is None, then return all subvolumes, sorted by path.
Only include subvolumes that contain at least one root pattern sourced from borgmatic
configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then
return all subvolumes instead, sorted by path.
Return the result as a sequence of matching subvolume mount points.
'''
@@ -73,7 +75,12 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None):
mount_point, candidate_patterns
),
)
if patterns is None or contained_patterns
if patterns is None
or any(
pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
for pattern in contained_patterns
)
)
return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path))
@@ -121,6 +128,7 @@ def make_snapshot_exclude_pattern(subvolume_path): # pragma: no cover
),
borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
borgmatic.borg.pattern.Pattern_style.FNMATCH,
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
@@ -153,6 +161,7 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
pattern.type,
pattern.style,
pattern.device,
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
@@ -198,7 +207,8 @@ def dump_data_sources(
dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}')
# Based on the configured patterns, determine Btrfs subvolumes to backup.
# Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those
# patterns that came from actual user configuration (as opposed to, say, other hooks).
btrfs_command = hook_config.get('btrfs_command', 'btrfs')
findmnt_command = hook_config.get('findmnt_command', 'findmnt')
subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns)
borgmatic/hooks/data_source/lvm.py
+39 -9
View File
@@ -1,5 +1,6 @@
import collections
import glob
import hashlib
import json
import logging
import os
@@ -33,7 +34,9 @@ def get_logical_volumes(lsblk_command, patterns=None):
between the current LVM logical volume mount points and the paths of any patterns. The idea is
that these pattern paths represent the requested logical volumes to snapshot.
If patterns is None, include all logical volume mounts points, not just those in patterns.
Only include logical volumes that contain at least one root pattern sourced from borgmatic
configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, include
all logical volume mounts points instead, not just those in patterns.
Return the result as a sequence of Logical_volume instances.
'''
@@ -72,7 +75,12 @@ def get_logical_volumes(lsblk_command, patterns=None):
device['mountpoint'], candidate_patterns
),
)
if not patterns or contained_patterns
if not patterns
or any(
pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
for pattern in contained_patterns
)
)
except KeyError as error:
raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"')
@@ -124,10 +132,14 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path): # prag
)
def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
MOUNT_POINT_HASH_LENGTH = 10
def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory):
'''
Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
path rewritten to be in a snapshot directory based on the given runtime directory.
Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume
containing it, return a new Pattern with its path rewritten to be in a snapshot directory based
on both the given runtime directory and the given Logical_volume's mount point.
Move any initial caret in a regular expression pattern path to the beginning, so as not to break
the regular expression.
@@ -142,6 +154,13 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
rewritten_path = initial_caret + os.path.join(
normalized_runtime_directory,
'lvm_snapshots',
# Including this hash prevents conflicts between snapshot patterns for different logical
# volumes. For instance, without this, snapshotting a logical volume at /var and another at
# /var/spool would result in overlapping snapshot patterns and therefore colliding mount
# attempts.
hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
MOUNT_POINT_HASH_LENGTH
),
'.', # Borg 1.4+ "slashdot" hack.
# Included so that the source directory ends up in the Borg archive at its "original" path.
pattern.path.lstrip('^').lstrip(os.path.sep),
@@ -152,6 +171,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
pattern.type,
pattern.style,
pattern.device,
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
@@ -180,7 +200,8 @@ def dump_data_sources(
dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
logger.info(f'Snapshotting LVM logical volumes{dry_run_label}')
# List logical volumes to get their mount points.
# List logical volumes to get their mount points, but only consider those patterns that came
# from actual user configuration (as opposed to, say, other hooks).
lsblk_command = hook_config.get('lsblk_command', 'lsblk')
requested_logical_volumes = get_logical_volumes(lsblk_command, patterns)
@@ -218,6 +239,9 @@ def dump_data_sources(
snapshot_mount_path = os.path.join(
normalized_runtime_directory,
'lvm_snapshots',
hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
MOUNT_POINT_HASH_LENGTH
),
logical_volume.mount_point.lstrip(os.path.sep),
)
@@ -233,7 +257,9 @@ def dump_data_sources(
)
for pattern in logical_volume.contained_patterns:
snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
snapshot_pattern = make_borg_snapshot_pattern(
pattern, logical_volume, normalized_runtime_directory
)
# Attempt to update the pattern in place, since pattern order matters to Borg.
try:
@@ -337,6 +363,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
os.path.normpath(borgmatic_runtime_directory),
),
'lvm_snapshots',
'*',
)
logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
umount_command = hook_config.get('umount_command', 'umount')
@@ -349,7 +376,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
snapshot_mount_path = os.path.join(
snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep)
)
if not os.path.isdir(snapshot_mount_path):
# If the snapshot mount path is empty, this is probably just a "shadow" of a nested
# logical volume and therefore there's nothing to unmount.
if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
continue
# This might fail if the directory is already mounted, but we swallow errors here since
@@ -374,7 +404,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
return
except subprocess.CalledProcessError as error:
logger.debug(error)
return
continue
if not dry_run:
shutil.rmtree(snapshots_directory)
borgmatic/hooks/data_source/mariadb.py
+2 -1
View File
@@ -216,7 +216,8 @@ def dump_data_sources(
if not dry_run:
patterns.append(
borgmatic.borg.pattern.Pattern(
os.path.join(borgmatic_runtime_directory, 'mariadb_databases')
os.path.join(borgmatic_runtime_directory, 'mariadb_databases'),
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
)
borgmatic/hooks/data_source/mongodb.py
+2 -1
View File
@@ -81,7 +81,8 @@ def dump_data_sources(
if not dry_run:
patterns.append(
borgmatic.borg.pattern.Pattern(
os.path.join(borgmatic_runtime_directory, 'mongodb_databases')
os.path.join(borgmatic_runtime_directory, 'mongodb_databases'),
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
)
borgmatic/hooks/data_source/mysql.py
+2 -1
View File
@@ -215,7 +215,8 @@ def dump_data_sources(
if not dry_run:
patterns.append(
borgmatic.borg.pattern.Pattern(
os.path.join(borgmatic_runtime_directory, 'mysql_databases')
os.path.join(borgmatic_runtime_directory, 'mysql_databases'),
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
)
borgmatic/hooks/data_source/postgresql.py
+2 -1
View File
@@ -239,7 +239,8 @@ def dump_data_sources(
if not dry_run:
patterns.append(
borgmatic.borg.pattern.Pattern(
os.path.join(borgmatic_runtime_directory, 'postgresql_databases')
os.path.join(borgmatic_runtime_directory, 'postgresql_databases'),
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
)
borgmatic/hooks/data_source/snapshot.py
+8 -6
View File
@@ -11,10 +11,10 @@ def get_contained_patterns(parent_directory, candidate_patterns):
paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then
/var is what we want to snapshot.
For this to work, a candidate pattern path can't have any globs or other non-literal characters
in the initial portion of the path that matches the parent directory. For instance, a parent
directory of /var would match a candidate pattern path of /var/log/*/data, but not a pattern
path like /v*/log/*/data.
For this function to work, a candidate pattern path can't have any globs or other non-literal
characters in the initial portion of the path that matches the parent directory. For instance, a
parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a
pattern path like /v*/log/*/data.
The one exception is that if a regular expression pattern path starts with "^", that will get
stripped off for purposes of matching against a parent directory.
@@ -31,8 +31,10 @@ def get_contained_patterns(parent_directory, candidate_patterns):
candidate
for candidate in candidate_patterns
for candidate_path in (pathlib.PurePath(candidate.path.lstrip('^')),)
if pathlib.PurePath(parent_directory) == candidate_path
or pathlib.PurePath(parent_directory) in candidate_path.parents
if (
pathlib.PurePath(parent_directory) == candidate_path
or pathlib.PurePath(parent_directory) in candidate_path.parents
)
)
candidate_patterns -= set(contained_patterns)
borgmatic/hooks/data_source/sqlite.py
+2 -1
View File
@@ -90,7 +90,8 @@ def dump_data_sources(
if not dry_run:
patterns.append(
borgmatic.borg.pattern.Pattern(
os.path.join(borgmatic_runtime_directory, 'sqlite_databases')
os.path.join(borgmatic_runtime_directory, 'sqlite_databases'),
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
)
borgmatic/hooks/data_source/zfs.py
+49 -11
View File
@@ -1,5 +1,6 @@
import collections
import glob
import hashlib
import logging
import os
import shutil
@@ -38,6 +39,9 @@ def get_datasets_to_backup(zfs_command, patterns):
pattern paths represent the requested datasets to snapshot. But also include any datasets tagged
with a borgmatic-specific user property, whether or not they appear in the patterns.
Only include datasets that contain at least one root pattern sourced from borgmatic
configuration (as opposed to generated elsewhere in borgmatic).
Return the result as a sequence of Dataset instances, sorted by mount point.
'''
list_output = borgmatic.execute.execute_command_and_capture_output(
@@ -48,7 +52,7 @@ def get_datasets_to_backup(zfs_command, patterns):
'-t',
'filesystem',
'-o',
f'name,mountpoint,{BORGMATIC_USER_PROPERTY}',
f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}',
)
)
@@ -60,7 +64,12 @@ def get_datasets_to_backup(zfs_command, patterns):
(
Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ())
for line in list_output.splitlines()
for (dataset_name, mount_point, user_property_value) in (line.rstrip().split('\t'),)
for (dataset_name, mount_point, can_mount, user_property_value) in (
line.rstrip().split('\t'),
)
# Skip datasets that are marked "canmount=off", because mounting their snapshots will
# result in completely empty mount points—thereby preventing us from backing them up.
if can_mount == 'on'
),
key=lambda dataset: dataset.mount_point,
reverse=True,
@@ -83,7 +92,12 @@ def get_datasets_to_backup(zfs_command, patterns):
for contained_patterns in (
(
(
(borgmatic.borg.pattern.Pattern(dataset.mount_point),)
(
borgmatic.borg.pattern.Pattern(
dataset.mount_point,
source=borgmatic.borg.pattern.Pattern_source.HOOK,
),
)
if dataset.auto_backup
else ()
)
@@ -92,7 +106,12 @@ def get_datasets_to_backup(zfs_command, patterns):
)
),
)
if contained_patterns
if dataset.auto_backup
or any(
pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG
for pattern in contained_patterns
)
),
key=lambda dataset: dataset.mount_point,
)
@@ -155,10 +174,14 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p
)
def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
MOUNT_POINT_HASH_LENGTH = 10
def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory):
'''
Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its
path rewritten to be in a snapshot directory based on the given runtime directory.
Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and the Dataset containing it,
return a new Pattern with its path rewritten to be in a snapshot directory based on both the
given runtime directory and the given Dataset's mount point.
Move any initial caret in a regular expression pattern path to the beginning, so as not to break
the regular expression.
@@ -173,6 +196,10 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
rewritten_path = initial_caret + os.path.join(
normalized_runtime_directory,
'zfs_snapshots',
# Including this hash prevents conflicts between snapshot patterns for different datasets.
# For instance, without this, snapshotting a dataset at /var and another at /var/spool would
# result in overlapping snapshot patterns and therefore colliding mount attempts.
hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
'.', # Borg 1.4+ "slashdot" hack.
# Included so that the source directory ends up in the Borg archive at its "original" path.
pattern.path.lstrip('^').lstrip(os.path.sep),
@@ -183,6 +210,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory):
pattern.type,
pattern.style,
pattern.device,
source=borgmatic.borg.pattern.Pattern_source.HOOK,
)
@@ -209,7 +237,8 @@ def dump_data_sources(
dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
logger.info(f'Snapshotting ZFS datasets{dry_run_label}')
# List ZFS datasets to get their mount points.
# List ZFS datasets to get their mount points, but only consider those patterns that came from
# actual user configuration (as opposed to, say, other hooks).
zfs_command = hook_config.get('zfs_command', 'zfs')
requested_datasets = get_datasets_to_backup(zfs_command, patterns)
@@ -234,6 +263,9 @@ def dump_data_sources(
snapshot_mount_path = os.path.join(
normalized_runtime_directory,
'zfs_snapshots',
hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(
MOUNT_POINT_HASH_LENGTH
),
dataset.mount_point.lstrip(os.path.sep),
)
@@ -249,7 +281,9 @@ def dump_data_sources(
)
for pattern in dataset.contained_patterns:
snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory)
snapshot_pattern = make_borg_snapshot_pattern(
pattern, dataset, normalized_runtime_directory
)
# Attempt to update the pattern in place, since pattern order matters to Borg.
try:
@@ -334,6 +368,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
os.path.normpath(borgmatic_runtime_directory),
),
'zfs_snapshots',
'*',
)
logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
umount_command = hook_config.get('umount_command', 'umount')
@@ -346,7 +381,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
# child datasets before the shorter mount point paths of parent datasets.
for mount_point in reversed(dataset_mount_points):
snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep))
if not os.path.isdir(snapshot_mount_path):
# If the snapshot mount path is empty, this is probably just a "shadow" of a nested
# dataset and therefore there's nothing to unmount.
if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
continue
# This might fail if the path is already mounted, but we swallow errors here since we'll
@@ -370,7 +408,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d
return
except subprocess.CalledProcessError as error:
logger.debug(error)
return
continue
if not dry_run:
shutil.rmtree(snapshots_directory)
docs/how-to/snapshot-your-filesystems.md
+11 -4
View File
@@ -54,8 +54,8 @@ You have a couple of options for borgmatic to find and backup your ZFS datasets:
* For any dataset you'd like backed up, add its mount point to borgmatic's
`source_directories` option.
* <span class="minilink minilink-addedin">New in version 1.9.6</span> Or
include the mount point with borgmatic's `patterns` or `patterns_from`
options.
include the mount point as a root pattern with borgmatic's `patterns` or
`patterns_from` options.
* Or set the borgmatic-specific user property
`org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs
set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find
@@ -65,6 +65,11 @@ If you have multiple borgmatic configuration files with ZFS enabled, and you'd
like particular datasets to be backed up only for particular configuration
files, use the `source_directories` option instead of the user property.
<span class="minilink minilink-addedin">New in version 1.9.11</span> borgmatic
won't snapshot datasets with the `canmount=off` property, which is often set on
datasets that only serve as a container for other datasets. Use `zfs get
canmount datasetname` to see the `canmount` value for a dataset.
During a backup, borgmatic automatically snapshots these discovered datasets
(non-recursively), temporarily mounts the snapshots within its [runtime
directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory),
@@ -147,7 +152,8 @@ For any subvolume you'd like backed up, add its path to borgmatic's
`source_directories` option.
<span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
the mount point with borgmatic's `patterns` or `patterns_from` options.
the mount point as a root pattern with borgmatic's `patterns` or `patterns_from`
options.
During a backup, borgmatic snapshots these subvolumes (non-recursively) and
includes the snapshotted files in the paths sent to Borg. borgmatic is also
@@ -252,7 +258,8 @@ For any logical volume you'd like backed up, add its mount point to
borgmatic's `source_directories` option.
<span class="minilink minilink-addedin">New in version 1.9.6</span> Or include
the mount point with borgmatic's `patterns` or `patterns_from` options.
the mount point as a root pattern with borgmatic's `patterns` or `patterns_from`
options.
During a backup, borgmatic automatically snapshots these discovered logical volumes
(non-recursively), temporarily mounts the snapshots within its [runtime
tests/end-to-end/commands/fake_zfs.py
+2
View File
@@ -27,6 +27,7 @@ BUILTIN_DATASETS = (
'used': '256K',
'avail': '23.7M',
'refer': '25K',
'canmount': 'on',
'mountpoint': '/pool',
},
{
@@ -34,6 +35,7 @@ BUILTIN_DATASETS = (
'used': '256K',
'avail': '23.7M',
'refer': '25K',
'canmount': 'on',
'mountpoint': '/pool/dataset',
},
)
tests/unit/actions/test_create.py
+21 -10
View File
@@ -5,16 +5,21 @@ import pytest
from flexmock import flexmock
from borgmatic.actions import create as module
from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
@pytest.mark.parametrize(
'pattern_line,expected_pattern',
(
('R /foo', Pattern('/foo')),
('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)),
('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)),
('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)),
('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)),
('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)),
('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)),
(
'+ sh:/foo*',
Pattern(
'/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG
),
),
),
)
def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern):
@@ -28,8 +33,8 @@ def test_parse_pattern_with_invalid_pattern_line_errors():
def test_collect_patterns_converts_source_directories():
assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == (
Pattern('/foo'),
Pattern('/bar'),
Pattern('/foo', source=Pattern_source.CONFIG),
Pattern('/bar', source=Pattern_source.CONFIG),
)
@@ -48,9 +53,15 @@ def test_collect_patterns_parses_config_patterns():
def test_collect_patterns_converts_exclude_patterns():
assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL),
Pattern(
'/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
),
Pattern(
'/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG
),
Pattern(
'**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG
),
)
tests/unit/borg/test_create.py
+3 -1
View File
@@ -4,7 +4,7 @@ import pytest
from flexmock import flexmock
from borgmatic.borg import create as module
from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
from ..test_verbosity import insert_logging_mock
@@ -663,6 +663,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
'/dev/null',
Pattern_type.NO_RECURSE,
Pattern_style.FNMATCH,
source=Pattern_source.INTERNAL,
),
),
'/run/borgmatic',
@@ -713,6 +714,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
'/dev/null',
Pattern_type.NO_RECURSE,
Pattern_style.FNMATCH,
source=Pattern_source.INTERNAL,
),
),
'/run/borgmatic',
tests/unit/hooks/data_source/test_btrfs.py
+70 -3
View File
@@ -1,7 +1,7 @@
import pytest
from flexmock import flexmock
from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
from borgmatic.hooks.data_source import btrfs as module
@@ -52,9 +52,14 @@ def test_get_subvolume_mount_points_with_findmnt_json_missing_filesystems_errors
def test_get_subvolumes_collects_subvolumes_matching_patterns():
flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2'))
contained_pattern = Pattern(
'/mnt1',
type=Pattern_type.ROOT,
source=Pattern_source.CONFIG,
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt1', object).and_return((Pattern('/mnt1'),))
).with_args('/mnt1', object).and_return((contained_pattern,))
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt2', object).and_return(())
@@ -66,7 +71,69 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns():
Pattern('/mnt1'),
Pattern('/mnt3'),
],
) == (module.Subvolume('/mnt1', contained_patterns=(Pattern('/mnt1'),)),)
) == (module.Subvolume('/mnt1', contained_patterns=(contained_pattern,)),)
def test_get_subvolumes_skips_non_root_patterns():
flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2'))
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt1', object).and_return(
(
Pattern(
'/mnt1',
type=Pattern_type.EXCLUDE,
source=Pattern_source.CONFIG,
),
)
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt2', object).and_return(())
assert (
module.get_subvolumes(
'btrfs',
'findmnt',
patterns=[
Pattern('/mnt1'),
Pattern('/mnt3'),
],
)
== ()
)
def test_get_subvolumes_skips_non_config_patterns():
flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2'))
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt1', object).and_return(
(
Pattern(
'/mnt1',
type=Pattern_type.ROOT,
source=Pattern_source.HOOK,
),
)
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt2', object).and_return(())
assert (
module.get_subvolumes(
'btrfs',
'findmnt',
patterns=[
Pattern('/mnt1'),
Pattern('/mnt3'),
],
)
== ()
)
def test_get_subvolumes_without_patterns_collects_all_subvolumes():
tests/unit/hooks/data_source/test_lvm.py
+358 -137
View File
@@ -1,7 +1,7 @@
import pytest
from flexmock import flexmock
from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
from borgmatic.hooks.data_source import lvm as module
@@ -37,14 +37,20 @@ def test_get_logical_volumes_filters_by_patterns():
}
'''
)
contained = {Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')}
contained = {
Pattern('/mnt/lvolume', source=Pattern_source.CONFIG),
Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG),
}
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args(None, contained).never()
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt/lvolume', contained).and_return(
(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir'))
(
Pattern('/mnt/lvolume', source=Pattern_source.CONFIG),
Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG),
)
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
@@ -54,17 +60,116 @@ def test_get_logical_volumes_filters_by_patterns():
).with_args('/mnt/notlvm', contained).never()
assert module.get_logical_volumes(
'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir'))
'lsblk',
patterns=(
Pattern('/mnt/lvolume', source=Pattern_source.CONFIG),
Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG),
),
) == (
module.Logical_volume(
name='vgroup-lvolume',
device_path='/dev/mapper/vgroup-lvolume',
mount_point='/mnt/lvolume',
contained_patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')),
contained_patterns=(
Pattern('/mnt/lvolume', source=Pattern_source.CONFIG),
Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG),
),
),
)
def test_get_logical_volumes_skips_non_root_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'''
{
"blockdevices": [
{
"name": "vgroup-lvolume",
"path": "/dev/mapper/vgroup-lvolume",
"mountpoint": "/mnt/lvolume",
"type": "lvm"
}
]
}
'''
)
contained = {
Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG),
Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG),
}
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args(None, contained).never()
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt/lvolume', contained).and_return(
(
Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG),
Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG),
)
)
assert (
module.get_logical_volumes(
'lsblk',
patterns=(
Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG),
Pattern(
'/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG
),
),
)
== ()
)
def test_get_logical_volumes_skips_non_config_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'''
{
"blockdevices": [
{
"name": "vgroup-lvolume",
"path": "/dev/mapper/vgroup-lvolume",
"mountpoint": "/mnt/lvolume",
"type": "lvm"
}
]
}
'''
)
contained = {
Pattern('/mnt/lvolume', source=Pattern_source.HOOK),
Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK),
}
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args(None, contained).never()
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/mnt/lvolume', contained).and_return(
(
Pattern('/mnt/lvolume', source=Pattern_source.HOOK),
Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK),
)
)
assert (
module.get_logical_volumes(
'lsblk',
patterns=(
Pattern('/mnt/lvolume', source=Pattern_source.HOOK),
Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK),
),
)
== ()
)
def test_get_logical_volumes_with_invalid_lsblk_json_errors():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
@@ -138,13 +243,13 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
(
(
Pattern('/foo/bar/baz'),
Pattern('/run/borgmatic/lvm_snapshots/./foo/bar/baz'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar/baz'),
),
(Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/./foo/bar')),
(Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar')),
(
Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
Pattern(
'^/run/borgmatic/lvm_snapshots/./foo/bar',
'^/run/borgmatic/lvm_snapshots/b33f/./foo/bar',
Pattern_type.INCLUDE,
Pattern_style.REGULAR_EXPRESSION,
),
@@ -152,40 +257,48 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
(
Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
Pattern(
'/run/borgmatic/lvm_snapshots/./foo/bar',
'/run/borgmatic/lvm_snapshots/b33f/./foo/bar',
Pattern_type.INCLUDE,
Pattern_style.REGULAR_EXPRESSION,
),
),
(Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/./foo')),
(Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/./')),
(Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')),
(Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')),
),
)
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
pattern, expected_pattern
):
assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
assert (
module.make_borg_snapshot_pattern(
pattern, flexmock(mount_point='/something'), '/run/borgmatic'
)
== expected_pattern
)
def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns():
config = {'lvm': {}}
patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
flexmock(module).should_receive('get_logical_volumes').and_return(
(
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
logical_volumes = (
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes)
flexmock(module.os).should_receive('getpid').and_return(1234)
flexmock(module).should_receive('snapshot_logical_volume').with_args(
'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE
@@ -203,18 +316,21 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns():
).and_return(
(module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),)
)
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
flexmock(module).should_receive('mount_snapshot').with_args(
'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1'
'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1'
).once()
flexmock(module).should_receive('mount_snapshot').with_args(
'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2'
).once()
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'))
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume2'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'))
assert (
module.dump_data_sources(
@@ -229,8 +345,8 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns():
)
assert patterns == [
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'),
]
@@ -259,22 +375,21 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots():
def test_dump_data_sources_uses_snapshot_size_for_snapshot():
config = {'lvm': {'snapshot_size': '1000PB'}}
patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
flexmock(module).should_receive('get_logical_volumes').and_return(
(
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
logical_volumes = (
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes)
flexmock(module.os).should_receive('getpid').and_return(1234)
flexmock(module).should_receive('snapshot_logical_volume').with_args(
'lvcreate',
@@ -298,18 +413,21 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot():
).and_return(
(module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),)
)
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
flexmock(module).should_receive('mount_snapshot').with_args(
'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1'
'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1'
).once()
flexmock(module).should_receive('mount_snapshot').with_args(
'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2'
).once()
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'))
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume2'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'))
assert (
module.dump_data_sources(
@@ -324,8 +442,8 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot():
)
assert patterns == [
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'),
]
@@ -339,22 +457,21 @@ def test_dump_data_sources_uses_custom_commands():
},
}
patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')]
flexmock(module).should_receive('get_logical_volumes').and_return(
(
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
logical_volumes = (
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes)
flexmock(module.os).should_receive('getpid').and_return(1234)
flexmock(module).should_receive('snapshot_logical_volume').with_args(
'/usr/local/bin/lvcreate',
@@ -378,18 +495,25 @@ def test_dump_data_sources_uses_custom_commands():
).and_return(
(module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),)
)
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
flexmock(module).should_receive('mount_snapshot').with_args(
'/usr/local/bin/mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1'
'/usr/local/bin/mount',
'/dev/lvolume1_snap',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).once()
flexmock(module).should_receive('mount_snapshot').with_args(
'/usr/local/bin/mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
'/usr/local/bin/mount',
'/dev/lvolume2_snap',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'))
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume2'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'))
assert (
module.dump_data_sources(
@@ -404,8 +528,8 @@ def test_dump_data_sources_uses_custom_commands():
)
assert patterns == [
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'),
]
@@ -463,22 +587,21 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte
def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns():
config = {'lvm': {}}
patterns = [Pattern('/hmm')]
flexmock(module).should_receive('get_logical_volumes').and_return(
(
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
logical_volumes = (
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes)
flexmock(module.os).should_receive('getpid').and_return(1234)
flexmock(module).should_receive('snapshot_logical_volume').with_args(
'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE
@@ -496,18 +619,21 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained
).and_return(
(module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),)
)
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
flexmock(module).should_receive('mount_snapshot').with_args(
'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1'
'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1'
).once()
flexmock(module).should_receive('mount_snapshot').with_args(
'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2'
'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2'
).once()
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'))
Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'))
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/lvolume2'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'))
Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'))
assert (
module.dump_data_sources(
@@ -523,8 +649,8 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained
assert patterns == [
Pattern('/hmm'),
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'),
]
@@ -694,16 +820,19 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume1',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).once()
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume2',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('get_snapshots').and_return(
(
@@ -799,9 +928,11 @@ def test_remove_data_source_dumps_with_missing_snapshot_directory_skips_unmount(
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots'
'/run/borgmatic/lvm_snapshots/b33f'
).and_return(False)
flexmock(module.shutil).should_receive('rmtree').never()
flexmock(module).should_receive('unmount_snapshot').never()
@@ -843,24 +974,92 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots'
'/run/borgmatic/lvm_snapshots/b33f'
).and_return(True)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots/mnt/lvolume1'
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1'
).and_return(False)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots/mnt/lvolume2'
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2'
).and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume1',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).never()
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume2',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('get_snapshots').and_return(
(
module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'),
module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'),
),
)
flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once()
flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once()
module.remove_data_source_dumps(
hook_config=config['lvm'],
config=config,
borgmatic_runtime_directory='/run/borgmatic',
dry_run=False,
)
def test_remove_data_source_dumps_with_empty_snapshot_mount_path_skips_unmount():
config = {'lvm': {}}
flexmock(module).should_receive('get_logical_volumes').and_return(
(
module.Logical_volume(
name='lvolume1',
device_path='/dev/lvolume1',
mount_point='/mnt/lvolume1',
contained_patterns=(Pattern('/mnt/lvolume1/subdir'),),
),
module.Logical_volume(
name='lvolume2',
device_path='/dev/lvolume2',
mount_point='/mnt/lvolume2',
contained_patterns=(Pattern('/mnt/lvolume2'),),
),
)
)
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots/b33f'
).and_return(True)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1'
).and_return(True)
flexmock(module.os).should_receive('listdir').with_args(
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1'
).and_return([])
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2'
).and_return(True)
flexmock(module.os).should_receive('listdir').with_args(
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2'
).and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).never()
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('get_snapshots').and_return(
(
@@ -900,24 +1099,27 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots'
'/run/borgmatic/lvm_snapshots/b33f'
).and_return(True)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots/mnt/lvolume1'
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1'
).and_return(True).and_return(False)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/lvm_snapshots/mnt/lvolume2'
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2'
).and_return(True).and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume1',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).never()
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume2',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('get_snapshots').and_return(
(
@@ -957,16 +1159,19 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume1',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).and_raise(FileNotFoundError)
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume2',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).never()
flexmock(module).should_receive('get_snapshots').never()
flexmock(module).should_receive('remove_snapshot').never()
@@ -979,7 +1184,7 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command():
)
def test_remove_data_source_dumps_bails_for_umount_command_error():
def test_remove_data_source_dumps_swallows_umount_command_error():
config = {'lvm': {}}
flexmock(module).should_receive('get_logical_volumes').and_return(
(
@@ -1000,19 +1205,28 @@ def test_remove_data_source_dumps_bails_for_umount_command_error():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume1',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).and_raise(module.subprocess.CalledProcessError(1, 'wtf'))
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume2',
).never()
flexmock(module).should_receive('get_snapshots').never()
flexmock(module).should_receive('remove_snapshot').never()
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('get_snapshots').and_return(
(
module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'),
module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'),
),
)
flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once()
flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once()
module.remove_data_source_dumps(
hook_config=config['lvm'],
@@ -1043,16 +1257,19 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume1',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).once()
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume2',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('get_snapshots').and_raise(FileNotFoundError)
flexmock(module).should_receive('remove_snapshot').never()
@@ -1086,16 +1303,19 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume1',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1',
).once()
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount',
'/run/borgmatic/lvm_snapshots/mnt/lvolume2',
'/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2',
).once()
flexmock(module).should_receive('get_snapshots').and_raise(
module.subprocess.CalledProcessError(1, 'wtf')
@@ -1133,6 +1353,7 @@ def test_remove_data_source_with_dry_run_skips_snapshot_unmount_and_delete():
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree').never()
flexmock(module).should_receive('unmount_snapshot').never()
flexmock(module).should_receive('get_snapshots').and_return(
tests/unit/hooks/data_source/test_zfs.py
+286 -69
View File
@@ -3,7 +3,7 @@ import os
import pytest
from flexmock import flexmock
from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type
from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type
from borgmatic.hooks.data_source import zfs as module
@@ -11,11 +11,19 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'dataset\t/dataset\t-\nother\t/other\t-',
'dataset\t/dataset\ton\t-\nother\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/dataset', object).and_return((Pattern('/dataset'),))
).with_args('/dataset', object).and_return(
(
Pattern(
'/dataset',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
)
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/other', object).and_return(())
@@ -23,24 +31,134 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns():
assert module.get_datasets_to_backup(
'zfs',
patterns=(
Pattern('/foo'),
Pattern('/dataset'),
Pattern('/bar'),
Pattern(
'/foo',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
Pattern(
'/dataset',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
Pattern(
'/bar',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
),
) == (
module.Dataset(
name='dataset',
mount_point='/dataset',
contained_patterns=(Pattern('/dataset'),),
contained_patterns=(
Pattern(
'/dataset',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
),
),
)
def test_get_datasets_to_backup_skips_non_root_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'dataset\t/dataset\ton\t-\nother\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/dataset', object).and_return(
(
Pattern(
'/dataset',
Pattern_type.EXCLUDE,
source=Pattern_source.CONFIG,
),
)
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/other', object).and_return(())
assert (
module.get_datasets_to_backup(
'zfs',
patterns=(
Pattern(
'/foo',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
Pattern(
'/dataset',
Pattern_type.EXCLUDE,
source=Pattern_source.CONFIG,
),
Pattern(
'/bar',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
),
)
== ()
)
def test_get_datasets_to_backup_skips_non_config_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'dataset\t/dataset\ton\t-\nother\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/dataset', object).and_return(
(
Pattern(
'/dataset',
Pattern_type.ROOT,
source=Pattern_source.HOOK,
),
)
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/other', object).and_return(())
assert (
module.get_datasets_to_backup(
'zfs',
patterns=(
Pattern(
'/foo',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
Pattern(
'/dataset',
Pattern_type.ROOT,
source=Pattern_source.HOOK,
),
Pattern(
'/bar',
Pattern_type.ROOT,
source=Pattern_source.CONFIG,
),
),
)
== ()
)
def test_get_datasets_to_backup_filters_datasets_by_user_property():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'dataset\t/dataset\tauto\nother\t/other\t-',
'dataset\t/dataset\ton\tauto\nother\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
@@ -49,16 +167,45 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property():
'get_contained_patterns'
).with_args('/other', object).and_return(())
assert module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar'))) == (
assert module.get_datasets_to_backup(
'zfs',
patterns=(Pattern('/foo'), Pattern('/bar')),
) == (
module.Dataset(
name='dataset',
mount_point='/dataset',
auto_backup=True,
contained_patterns=(Pattern('/dataset'),),
contained_patterns=(Pattern('/dataset', source=Pattern_source.HOOK),),
),
)
def test_get_datasets_to_backup_filters_datasets_by_canmount_property():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).and_return(
'dataset\t/dataset\toff\t-\nother\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/dataset', object).and_return((Pattern('/dataset'),))
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns'
).with_args('/other', object).and_return(())
assert (
module.get_datasets_to_backup(
'zfs',
patterns=(
Pattern('/foo'),
Pattern('/dataset'),
Pattern('/bar'),
),
)
== ()
)
def test_get_datasets_to_backup_with_invalid_list_output_raises():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
@@ -94,13 +241,13 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets():
(
(
Pattern('/foo/bar/baz'),
Pattern('/run/borgmatic/zfs_snapshots/./foo/bar/baz'),
Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar/baz'),
),
(Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/./foo/bar')),
(Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar')),
(
Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
Pattern(
'^/run/borgmatic/zfs_snapshots/./foo/bar',
'^/run/borgmatic/zfs_snapshots/b33f/./foo/bar',
Pattern_type.INCLUDE,
Pattern_style.REGULAR_EXPRESSION,
),
@@ -108,46 +255,55 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets():
(
Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION),
Pattern(
'/run/borgmatic/zfs_snapshots/./foo/bar',
'/run/borgmatic/zfs_snapshots/b33f/./foo/bar',
Pattern_type.INCLUDE,
Pattern_style.REGULAR_EXPRESSION,
),
),
(Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/./foo')),
(Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/./')),
(Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')),
(Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')),
),
)
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
pattern, expected_pattern
):
assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
assert (
module.make_borg_snapshot_pattern(
pattern, flexmock(mount_point='/something'), '/run/borgmatic'
)
== expected_pattern
)
def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns():
flexmock(module).should_receive('get_datasets_to_backup').and_return(
(
flexmock(
name='dataset',
mount_point='/mnt/dataset',
contained_patterns=(Pattern('/mnt/dataset/subdir'),),
)
)
dataset = flexmock(
name='dataset',
mount_point='/mnt/dataset',
contained_patterns=(Pattern('/mnt/dataset/subdir'),),
)
flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,))
flexmock(module.os).should_receive('getpid').and_return(1234)
full_snapshot_name = 'dataset@borgmatic-1234'
flexmock(module).should_receive('snapshot_dataset').with_args(
'zfs',
full_snapshot_name,
).once()
snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset'
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset'
flexmock(module).should_receive('mount_snapshot').with_args(
'mount',
full_snapshot_name,
module.os.path.normpath(snapshot_mount_path),
).once()
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/dataset/subdir'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir'))
Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir'))
patterns = [Pattern('/mnt/dataset/subdir')]
assert (
@@ -188,30 +344,30 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots():
def test_dump_data_sources_uses_custom_commands():
flexmock(module).should_receive('get_datasets_to_backup').and_return(
(
flexmock(
name='dataset',
mount_point='/mnt/dataset',
contained_patterns=(Pattern('/mnt/dataset/subdir'),),
)
)
dataset = flexmock(
name='dataset',
mount_point='/mnt/dataset',
contained_patterns=(Pattern('/mnt/dataset/subdir'),),
)
flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,))
flexmock(module.os).should_receive('getpid').and_return(1234)
full_snapshot_name = 'dataset@borgmatic-1234'
flexmock(module).should_receive('snapshot_dataset').with_args(
'/usr/local/bin/zfs',
full_snapshot_name,
).once()
snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset'
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset'
flexmock(module).should_receive('mount_snapshot').with_args(
'/usr/local/bin/mount',
full_snapshot_name,
module.os.path.normpath(snapshot_mount_path),
).once()
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/dataset/subdir'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir'))
Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir'))
patterns = [Pattern('/mnt/dataset/subdir')]
hook_config = {
'zfs_command': '/usr/local/bin/zfs',
@@ -261,30 +417,30 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patter
def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns():
flexmock(module).should_receive('get_datasets_to_backup').and_return(
(
flexmock(
name='dataset',
mount_point='/mnt/dataset',
contained_patterns=(Pattern('/mnt/dataset/subdir'),),
)
)
dataset = flexmock(
name='dataset',
mount_point='/mnt/dataset',
contained_patterns=(Pattern('/mnt/dataset/subdir'),),
)
flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,))
flexmock(module.os).should_receive('getpid').and_return(1234)
full_snapshot_name = 'dataset@borgmatic-1234'
flexmock(module).should_receive('snapshot_dataset').with_args(
'zfs',
full_snapshot_name,
).once()
snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset'
flexmock(module.hashlib).should_receive('shake_256').and_return(
flexmock(hexdigest=lambda length: 'b33f')
)
snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset'
flexmock(module).should_receive('mount_snapshot').with_args(
'mount',
full_snapshot_name,
module.os.path.normpath(snapshot_mount_path),
).once()
flexmock(module).should_receive('make_borg_snapshot_pattern').with_args(
Pattern('/mnt/dataset/subdir'), '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir'))
Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic'
).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir'))
patterns = [Pattern('/hmm')]
assert (
@@ -317,11 +473,14 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'umount', '/run/borgmatic/zfs_snapshots/mnt/dataset'
'umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).once()
flexmock(module).should_receive('get_all_snapshots').and_return(
('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid')
@@ -343,11 +502,14 @@ def test_remove_data_source_dumps_use_custom_commands():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset'
'/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).once()
flexmock(module).should_receive('get_all_snapshots').and_return(
('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid')
@@ -416,11 +578,14 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset'
'/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).and_raise(FileNotFoundError)
flexmock(module).should_receive('get_all_snapshots').never()
flexmock(module).should_receive('destroy_snapshot').never()
@@ -434,19 +599,26 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command():
)
def test_remove_data_source_dumps_bails_for_umount_command_error():
def test_remove_data_source_dumps_swallows_umount_command_error():
flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').with_args(
'/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset'
'/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).and_raise(module.subprocess.CalledProcessError(1, 'wtf'))
flexmock(module).should_receive('get_all_snapshots').never()
flexmock(module).should_receive('destroy_snapshot').never()
flexmock(module).should_receive('get_all_snapshots').and_return(
('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid')
)
flexmock(module).should_receive('destroy_snapshot').with_args(
'/usr/local/bin/zfs', 'dataset@borgmatic-1234'
).once()
hook_config = {'zfs_command': '/usr/local/bin/zfs', 'umount_command': '/usr/local/bin/umount'}
module.remove_data_source_dumps(
@@ -462,7 +634,9 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_no
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(False)
flexmock(module.shutil).should_receive('rmtree').never()
flexmock(module).should_receive('unmount_snapshot').never()
@@ -486,13 +660,50 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/zfs_snapshots'
'/run/borgmatic/zfs_snapshots/b33f'
).and_return(True)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/zfs_snapshots/mnt/dataset'
'/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).and_return(False)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').never()
flexmock(module).should_receive('get_all_snapshots').and_return(
('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid')
)
flexmock(module).should_receive('destroy_snapshot').with_args(
'zfs', 'dataset@borgmatic-1234'
).once()
module.remove_data_source_dumps(
hook_config={},
config={'source_directories': '/mnt/dataset', 'zfs': {}},
borgmatic_runtime_directory='/run/borgmatic',
dry_run=False,
)
def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_empty():
flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/zfs_snapshots/b33f'
).and_return(True)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).and_return(True)
flexmock(module.os).should_receive('listdir').with_args(
'/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).and_return([])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').never()
flexmock(module).should_receive('get_all_snapshots').and_return(
@@ -515,13 +726,16 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/zfs_snapshots'
'/run/borgmatic/zfs_snapshots/b33f'
).and_return(True)
flexmock(module.os.path).should_receive('isdir').with_args(
'/run/borgmatic/zfs_snapshots/mnt/dataset'
'/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
).and_return(True).and_return(False)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree')
flexmock(module).should_receive('unmount_snapshot').never()
flexmock(module).should_receive('get_all_snapshots').and_return(
@@ -544,8 +758,11 @@ def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy():
flexmock(module.borgmatic.config.paths).should_receive(
'replace_temporary_subdirectory_with_glob'
).and_return('/run/borgmatic')
flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path])
flexmock(module.glob).should_receive('glob').replace_with(
lambda path: [path.replace('*', 'b33f')]
)
flexmock(module.os.path).should_receive('isdir').and_return(True)
flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
flexmock(module.shutil).should_receive('rmtree').never()
flexmock(module).should_receive('unmount_snapshot').never()
flexmock(module).should_receive('get_all_snapshots').and_return(