diff --git a/NEWS b/NEWS index c9a55a51..e8f686ff 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 1.9.4.dev0 + * #251 (beta): Add a Btrfs hook for snapshotting and backing up Btrfs subvolumes. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/ * #926: Fix library error when running within a PyInstaller bundle. * Reorganize data source and monitoring hooks to make developing new hooks easier. diff --git a/borgmatic/config/paths.py b/borgmatic/config/paths.py index 5482efef..ecc64b3b 100644 --- a/borgmatic/config/paths.py +++ b/borgmatic/config/paths.py @@ -33,10 +33,11 @@ def get_borgmatic_source_directory(config): TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-' -def replace_temporary_subdirectory_with_glob(path): +def replace_temporary_subdirectory_with_glob(path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX): ''' - Given an absolute temporary directory path, look for a subdirectory within it starting with the - temporary directory prefix and replace it with an appropriate glob. For instance, given: + Given an absolute temporary directory path and an optional temporary directory prefix, look for + a subdirectory within it starting with the temporary directory prefix (or a default) and replace + it with an appropriate glob. For instance, given: /tmp/borgmatic-aet8kn93/borgmatic @@ -50,8 +51,8 @@ def replace_temporary_subdirectory_with_glob(path): '/', *( ( - f'{TEMPORARY_DIRECTORY_PREFIX}*' - if subdirectory.startswith(TEMPORARY_DIRECTORY_PREFIX) + f'{temporary_directory_prefix}*' + if subdirectory.startswith(temporary_directory_prefix) else subdirectory ) for subdirectory in path.split(os.path.sep) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ddf9aaeb..9ba69a13 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2288,3 +2288,17 @@ properties: example: /usr/local/bin/umount description: | Configuration for integration with the ZFS filesystem. + btrfs: + type: ["object", "null"] + additionalProperties: false + properties: + btrfs_command: + type: string + description: | + Command to use instead of "btrfs". + example: /usr/local/bin/btrfs + findmnt_command: + type: string + description: | + Command to use instead of "findmnt". + example: /usr/local/bin/findmnt diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py new file mode 100644 index 00000000..0f7f46de --- /dev/null +++ b/borgmatic/hooks/data_source/btrfs.py @@ -0,0 +1,283 @@ +import functools +import glob +import logging +import os +import shutil +import subprocess + +import borgmatic.config.paths +import borgmatic.execute + +logger = logging.getLogger(__name__) + + +def use_streaming(hook_config, config, log_prefix): # pragma: no cover + ''' + Return whether dump streaming is used for this hook. (Spoiler: It isn't.) + ''' + return False + + +def get_subvolumes(btrfs_command, findmnt_command, source_directories=None): + ''' + Given a Btrfs command to run and a sequence of configured source directories, find the + intersection between the current Btrfs filesystem and subvolume mount points and the configured + borgmatic source directories. The idea is that these are the requested subvolumes to snapshot. + + If the source directories is None, then return all subvolumes. + + Return the result as a sequence of matching subvolume mount points. + ''' + # Find all top-level Btrfs filesystem mount points. + findmnt_output = borgmatic.execute.execute_command_and_capture_output( + ( + findmnt_command, + '-nt', + 'btrfs', + ) + ) + + try: + filesystem_mount_points = tuple( + line.rstrip().split(' ')[0] + for line in findmnt_output.splitlines() + ) + except ValueError: + raise ValueError('Invalid {findmnt_command} output') + + source_directories_set = set(source_directories or ()) + subvolumes = [] + + # For each filesystem mount point, find its subvolumes and match them again the given source + # directories to find the subvolumes to backup. Also try to match the filesystem mount point + # itself. + for mount_point in filesystem_mount_points: + if source_directories is None or mount_point in source_directories_set: + subvolumes.append(mount_point) + + btrfs_output = borgmatic.execute.execute_command_and_capture_output( + ( + btrfs_command, + 'subvolume', + 'list', + mount_point, + ) + ) + + try: + subvolumes.extend( + subvolume_path + for line in btrfs_output.splitlines() + for subvolume_subpath in (line.rstrip().split(' ')[-1],) + for subvolume_path in (os.path.join(mount_point, subvolume_subpath),) + if source_directories is None or subvolume_path in source_directories_set + ) + except ValueError: + raise ValueError('Invalid {btrfs_command} subvolume list output') + + return tuple(subvolumes) + + +BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot-' + + +def make_snapshot_path(subvolume_path): # pragma: no cover + ''' + Given the path to a subvolume, make a corresponding snapshot path for it. + ''' + return os.path.join( + subvolume_path, + f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}', + '.', # Borg 1.4+ "slashdot" hack. + # Included so that the snapshot ends up in the Borg archive at the "original" subvolume + # path. + subvolume_path.lstrip(os.path.sep), + ) + + +def make_snapshot_exclude_path(subvolume_path): # pragma: no cover + ''' + Given the path to a subvolume, make a corresponding exclude path for its embedded snapshot path. + This is to work around a quirk of Btrfs: If you make a snapshot path as a child directory of a + subvolume, then the snapshot's own initial directory component shows up as an empty directory + within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and make a + snapshot of it at: + + /mnt/.borgmatic-snapshot-1234/mnt + + ... then the snapshot itself will have an empty directory at: + + /mnt/.borgmatic-snapshot-1234/mnt/.borgmatic-snapshot-1234 + + So to prevent that from ending up in the Borg archive, this function produces its path for + exclusion. + ''' + snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' + + return os.path.join( + subvolume_path, + snapshot_directory, + subvolume_path.lstrip(os.path.sep), + snapshot_directory, + ) + + +def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path): # pragma: no cover + ''' + Given a Btrfs command to run, the path to a subvolume, and the path for a snapshot, create a new + Btrfs snapshot of the subvolume. + ''' + os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True) + + borgmatic.execute.execute_command( + ( + btrfs_command, + 'subvolume', + 'snapshot', + subvolume_path, + snapshot_path, + ), + output_log_level=logging.DEBUG, + ) + + +def dump_data_sources( + hook_config, + config, + log_prefix, + config_paths, + borgmatic_runtime_directory, + source_directories, + dry_run, +): + ''' + Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic + configuration file paths, the borgmatic runtime directory, the configured source directories, + and whether this is a dry run, auto-detect and snapshot any Btrfs subvolume mount points listed + in the given source directories. Also update those source directories, replacing subvolume mount + points with corresponding snapshot directories so they get stored in the Borg archive instead. + Use the log prefix in any log entries. + + Return an empty sequence, since there are no ongoing dump processes from this hook. + + If this is a dry run, then don't actually snapshot anything. + ''' + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'{log_prefix}: Snapshotting Btrfs datasets{dry_run_label}') + + # Based on the configured source directories, determine Btrfs subvolumes to backup. + btrfs_command = hook_config.get('btrfs_command', 'btrfs') + findmnt_command = hook_config.get('findmnt_command', 'findmnt') + requested_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command, source_directories) + + # Snapshot each subvolume, rewriting source directories to use their snapshot paths. + for subvolume_path in requested_subvolume_paths: + logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume_path} subvolume') + + snapshot_path = make_snapshot_path(subvolume_path) + + if dry_run: + continue + + snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path) + + if subvolume_path in source_directories: + source_directories.remove(subvolume_path) + + source_directories.append(snapshot_path) + config.setdefault('exclude_patterns', []).append(make_snapshot_exclude_path(subvolume_path)) + + return [] + + +def delete_snapshot(btrfs_command, snapshot_path): # pragma: no cover + ''' + Given a Btrfs command to run and the name of a snapshot path, delete it. + ''' + borgmatic.execute.execute_command( + ( + btrfs_command, + 'subvolume', + 'delete', + snapshot_path, + ), + output_log_level=logging.DEBUG, + ) + + +def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run): + ''' + Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic runtime + directory, and whether this is a dry run, delete any Btrfs snapshots created by borgmatic. Use + the log prefix in any log entries. If this is a dry run, then don't actually remove anything. + ''' + dry_run_label = ' (dry run; not actually removing anything)' if dry_run else '' + + btrfs_command = hook_config.get('btrfs_command', 'btrfs') + findmnt_command = hook_config.get('findmnt_command', 'findmnt') + + try: + all_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command) + except FileNotFoundError as error: + logger.debug(f'{log_prefix}: Could not find "{error.filename}" command') + return + except subprocess.CalledProcessError as error: + logger.debug(f'{log_prefix}: {error}') + return + + for subvolume_path in all_subvolume_paths: + subvolume_snapshots_glob = borgmatic.config.paths.replace_temporary_subdirectory_with_glob( + os.path.normpath(make_snapshot_path(subvolume_path)), + temporary_directory_prefix=BORGMATIC_SNAPSHOT_PREFIX, + ) + + logger.debug( + f'{log_prefix}: Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}' + ) + + for snapshot_path in glob.glob(subvolume_snapshots_glob): + if not os.path.isdir(snapshot_path): + continue + + logger.debug(f'{log_prefix}: Deleting Btrfs snapshot {snapshot_path}{dry_run_label}') + + if dry_run: + continue + + try: + delete_snapshot(btrfs_command, snapshot_path) + except FileNotFoundError: + logger.debug(f'{log_prefix}: Could not find "{btrfs_command}" command') + return + except subprocess.CalledProcessError as error: + logger.debug(f'{log_prefix}: {error}') + return + + # Strip off the subvolume path from the end of the snapshot path and then delete the + # resulting directory. + shutil.rmtree(snapshot_path.rsplit(subvolume_path, 1)[0]) + + +def make_data_source_dump_patterns( + hook_config, config, log_prefix, borgmatic_runtime_directory, name=None +): # pragma: no cover + ''' + Restores aren't implemented, because stored files can be extracted directly with "extract". + ''' + return () + + +def restore_data_source_dump( + hook_config, + config, + log_prefix, + data_source, + dry_run, + extract_process, + connection_params, + borgmatic_runtime_directory, +): # pragma: no cover + ''' + Restores aren't implemented, because stored files can be extracted directly with "extract". + ''' + raise NotImplementedError() diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 6e000674..a9e44c9d 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -104,6 +104,7 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p first). ''' os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True) + borgmatic.execute.execute_command( ( mount_command, @@ -131,8 +132,7 @@ def dump_data_sources( is a dry run, auto-detect and snapshot any ZFS dataset mount points listed in the given source directories and any dataset with a borgmatic-specific user property. Also update those source directories, replacing dataset mount points with corresponding snapshot directories so they get - stored in the Borg archive instead of the dataset mount points. Use the log prefix in any log - entries. + stored in the Borg archive instead. Use the log prefix in any log entries. Return an empty sequence, since there are no ongoing dump processes from this hook.