From f1c5f114228cd3a698e3a149d589fed1fc879503 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 5 Dec 2024 11:18:53 -0800 Subject: [PATCH] Add an end-to-end test for the ZFS hook using a fake ZFS binary (#80). --- borgmatic/hooks/data_source/btrfs.py | 16 ++-- borgmatic/hooks/data_source/lvm.py | 26 +++--- borgmatic/hooks/data_source/zfs.py | 30 +++---- tests/end-to-end/commands/__init__.py | 0 tests/end-to-end/commands/fake_mount.py | 26 ++++++ tests/end-to-end/commands/fake_umount.py | 22 +++++ tests/end-to-end/commands/fake_zfs.py | 103 +++++++++++++++++++++++ tests/end-to-end/test_zfs.py | 62 ++++++++++++++ 8 files changed, 249 insertions(+), 36 deletions(-) create mode 100644 tests/end-to-end/commands/__init__.py create mode 100755 tests/end-to-end/commands/fake_mount.py create mode 100755 tests/end-to-end/commands/fake_umount.py create mode 100755 tests/end-to-end/commands/fake_zfs.py create mode 100644 tests/end-to-end/test_zfs.py diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 5302c6cd..c99ee04b 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -24,8 +24,8 @@ def get_filesystem_mount_points(findmnt_command): Given a findmnt command to run, get all top-level Btrfs filesystem mount points. ''' findmnt_output = borgmatic.execute.execute_command_and_capture_output( - ( - findmnt_command, + tuple(findmnt_command.split(' ')) + + ( '-nt', 'btrfs', ) @@ -40,8 +40,8 @@ def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point): that filesystem. Include the filesystem itself. ''' btrfs_output = borgmatic.execute.execute_command_and_capture_output( - ( - btrfs_command, + tuple(btrfs_command.split(' ')) + + ( 'subvolume', 'list', filesystem_mount_point, @@ -167,8 +167,8 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path): # pragma: os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True) borgmatic.execute.execute_command( - ( - btrfs_command, + tuple(btrfs_command.split(' ')) + + ( 'subvolume', 'snapshot', '-r', # Read-only, @@ -242,8 +242,8 @@ def delete_snapshot(btrfs_command, snapshot_path): # pragma: no cover Given a Btrfs command to run and the name of a snapshot path, delete it. ''' borgmatic.execute.execute_command( - ( - btrfs_command, + tuple(btrfs_command.split(' ')) + + ( 'subvolume', 'delete', snapshot_path, diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 717f60ec..48418f43 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -51,7 +51,7 @@ def get_logical_volumes(lsblk_command, source_directories=None): ) ) except json.JSONDecodeError as error: - raise ValueError('Invalid {lsblk_command} JSON output: {error}') + raise ValueError(f'Invalid {lsblk_command} JSON output: {error}') candidate_source_directories = set(source_directories or ()) @@ -84,8 +84,8 @@ def snapshot_logical_volume( snapshot, and a snapshot size string, create a new LVM snapshot. ''' borgmatic.execute.execute_command( - ( - lvcreate_command, + tuple(lvcreate_command.split(' ')) + + ( '--snapshot', ('--extents' if '%' in snapshot_size else '--size'), snapshot_size, @@ -106,8 +106,8 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path): # prag os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True) borgmatic.execute.execute_command( - ( - mount_command, + tuple(mount_command.split(' ')) + + ( '-o', 'ro', snapshot_device, @@ -221,8 +221,8 @@ def unmount_snapshot(umount_command, snapshot_mount_path): # pragma: no cover Given a umount command to run and the mount path of a snapshot, unmount it. ''' borgmatic.execute.execute_command( - ( - umount_command, + tuple(umount_command.split(' ')) + + ( snapshot_mount_path, ), output_log_level=logging.DEBUG, @@ -234,8 +234,8 @@ def delete_snapshot(lvremove_command, snapshot_device_path): # pragma: no cover Given an lvremove command to run and the device path of a snapshot, remove it it. ''' borgmatic.execute.execute_command( - ( - lvremove_command, + tuple(lvremove_command.split(' ')) + + ( '--force', # Suppress an interactive "are you sure?" type prompt. snapshot_device_path, ), @@ -258,9 +258,9 @@ def get_snapshots(lvs_command, snapshot_name=None): try: snapshot_info = json.loads( borgmatic.execute.execute_command_and_capture_output( - ( - # Use lvs instead of lsblk here because lsblk can't filter to just snapshots. - lvs_command, + # Use lvs instead of lsblk here because lsblk can't filter to just snapshots. + tuple(lvs_command.split(' ')) + + ( '--report-format', 'json', '--options', @@ -271,7 +271,7 @@ def get_snapshots(lvs_command, snapshot_name=None): ) ) except json.JSONDecodeError as error: - raise ValueError('Invalid {lvs_command} JSON output: {error}') + raise ValueError(f'Invalid {lvs_command} JSON output: {error}') try: return tuple( diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 794712ae..d5dac5b0 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -41,8 +41,8 @@ def get_datasets_to_backup(zfs_command, source_directories): Return the result as a sequence of Dataset instances, sorted by mount point. ''' list_output = borgmatic.execute.execute_command_and_capture_output( - ( - zfs_command, + tuple(zfs_command.split(' ')) + + ( 'list', '-H', '-t', @@ -67,7 +67,7 @@ def get_datasets_to_backup(zfs_command, source_directories): reverse=True, ) except ValueError: - raise ValueError('Invalid {zfs_command} list output') + raise ValueError(f'Invalid {zfs_command} list output') candidate_source_directories = set(source_directories) @@ -102,8 +102,8 @@ def get_all_dataset_mount_points(zfs_command): Given a ZFS command to run, return all ZFS datasets as a sequence of sorted mount points. ''' list_output = borgmatic.execute.execute_command_and_capture_output( - ( - zfs_command, + tuple(zfs_command.split(' ')) + + ( 'list', '-H', '-t', @@ -122,8 +122,8 @@ def snapshot_dataset(zfs_command, full_snapshot_name): # pragma: no cover snapshot. ''' borgmatic.execute.execute_command( - ( - zfs_command, + tuple(zfs_command.split(' ')) + + ( 'snapshot', full_snapshot_name, ), @@ -140,8 +140,8 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True) borgmatic.execute.execute_command( - ( - mount_command, + tuple(mount_command.split(' ')) + + ( '-t', 'zfs', full_snapshot_name, @@ -237,8 +237,8 @@ def unmount_snapshot(umount_command, snapshot_mount_path): # pragma: no cover Given a umount command to run and the mount path of a snapshot, unmount it. ''' borgmatic.execute.execute_command( - ( - umount_command, + tuple(umount_command.split(' ')) + + ( snapshot_mount_path, ), output_log_level=logging.DEBUG, @@ -251,8 +251,8 @@ def destroy_snapshot(zfs_command, full_snapshot_name): # pragma: no cover it. ''' borgmatic.execute.execute_command( - ( - zfs_command, + tuple(zfs_command.split(' ')) + + ( 'destroy', full_snapshot_name, ), @@ -266,8 +266,8 @@ def get_all_snapshots(zfs_command): form "dataset@snapshot". ''' list_output = borgmatic.execute.execute_command_and_capture_output( - ( - zfs_command, + tuple(zfs_command.split(' ')) + + ( 'list', '-H', '-t', diff --git a/tests/end-to-end/commands/__init__.py b/tests/end-to-end/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/end-to-end/commands/fake_mount.py b/tests/end-to-end/commands/fake_mount.py new file mode 100755 index 00000000..21e161c9 --- /dev/null +++ b/tests/end-to-end/commands/fake_mount.py @@ -0,0 +1,26 @@ +import argparse +import os +import sys + + +def parse_arguments(*unparsed_arguments): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('-t', dest='type') + parser.add_argument('snapshot_name') + parser.add_argument('mount_point') + + return parser.parse_args(unparsed_arguments) + + +def main(): + arguments = parse_arguments(*sys.argv[1:]) + + subdirectory = os.path.join(arguments.mount_point, 'subdir') + os.mkdir(subdirectory) + test_file = open(os.path.join(subdirectory, 'file.txt'), 'w') + test_file.write('contents') + test_file.close() + + +if __name__ == '__main__': + main() diff --git a/tests/end-to-end/commands/fake_umount.py b/tests/end-to-end/commands/fake_umount.py new file mode 100755 index 00000000..e386c802 --- /dev/null +++ b/tests/end-to-end/commands/fake_umount.py @@ -0,0 +1,22 @@ +import argparse +import os +import shutil +import sys + + +def parse_arguments(*unparsed_arguments): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('mount_point') + + return parser.parse_args(unparsed_arguments) + + +def main(): + arguments = parse_arguments(*sys.argv[1:]) + + subdirectory = os.path.join(arguments.mount_point, 'subdir') + shutil.rmtree(subdirectory) + + +if __name__ == '__main__': + main() diff --git a/tests/end-to-end/commands/fake_zfs.py b/tests/end-to-end/commands/fake_zfs.py new file mode 100755 index 00000000..88c4d1e3 --- /dev/null +++ b/tests/end-to-end/commands/fake_zfs.py @@ -0,0 +1,103 @@ +import argparse +import json +import sys + + +def parse_arguments(*unparsed_arguments): + global_parser = argparse.ArgumentParser(add_help=False) + action_parsers = global_parser.add_subparsers(dest='action') + + list_parser = action_parsers.add_parser('list') + list_parser.add_argument('-H', dest='header', action='store_false', default=True) + list_parser.add_argument('-t', dest='type', default='filesystem') + list_parser.add_argument('-o', dest='properties', default='name,used,avail,refer,mountpoint') + + snapshot_parser = action_parsers.add_parser('snapshot') + snapshot_parser.add_argument('name') + + destroy_parser = action_parsers.add_parser('destroy') + destroy_parser.add_argument('name') + + return global_parser.parse_args(unparsed_arguments) + + +BUILTIN_DATASETS = ( + { + 'name': 'pool', + 'used': '256K', + 'avail': '23.7M', + 'refer': '25K', + 'mountpoint': '/pool', + }, + { + 'name': 'pool/dataset', + 'used': '256K', + 'avail': '23.7M', + 'refer': '25K', + 'mountpoint': '/pool/dataset', + }, +) + + + + + +def load_snapshots(): + try: + return json.load(open('/tmp/fake_zfs.json')) + except FileNotFoundError: + return [] + + +def save_snapshots(snapshots): + json.dump(snapshots, open('/tmp/fake_zfs.json', 'w')) + + +def print_dataset_list(arguments, datasets, snapshots): + properties = arguments.properties.split(',') + data = ( + ( + tuple(property_name.upper() for property_name in properties), + ) + if arguments.header else () + ) + tuple( + tuple( + dataset.get(property_name, '-') for property_name in properties + ) + for dataset in (snapshots if arguments.type == 'snapshot' else datasets) + ) + + if not data: + return + + for data_row in data: + print('\t'.join(data_row)) + + +def main(): + arguments = parse_arguments(*sys.argv[1:]) + snapshots = load_snapshots() + + if arguments.action == 'list': + print_dataset_list(arguments, BUILTIN_DATASETS, snapshots) + elif arguments.action == 'snapshot': + snapshots.append( + { + 'name': arguments.name, + 'used': '0B', + 'avail': '-', + 'refer': '25K', + 'mountpoint': '-', + }, + ) + save_snapshots(snapshots) + elif arguments.action == 'destroy': + snapshots = [ + snapshot for snapshot in snapshots + if snapshot['name'] != arguments.name + ] + save_snapshots(snapshots) + + +if __name__ == '__main__': + main() diff --git a/tests/end-to-end/test_zfs.py b/tests/end-to-end/test_zfs.py new file mode 100644 index 00000000..9a173157 --- /dev/null +++ b/tests/end-to-end/test_zfs.py @@ -0,0 +1,62 @@ +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing (including injecting the given repository path and tacking on an encryption + passphrase). + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '- /pool/dataset') + .replace('- /var/log/syslog*', '') + + 'encryption_passphrase: "test"\n' + + 'zfs:\n' + + ' zfs_command: python3 /app/tests/end-to-end/commands/fake_zfs.py\n' + + ' mount_command: python3 /app/tests/end-to-end/commands/fake_mount.py\n' + + ' umount_command: python3 /app/tests/end-to-end/commands/fake_umount.py' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_zfs_create_and_list(): + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path) + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' ') + ) + + # Run a create action to exercise ZFS snapshotting and backup. + subprocess.check_call(f'borgmatic --config {config_path} create'.split(' ')) + + # List the resulting archive and assert that the snapshotted files are there. + output = subprocess.check_output( + f'borgmatic --config {config_path} list --archive latest'.split(' ') + ).decode(sys.stdout.encoding) + + assert 'pool/dataset/subdir/file.txt' in output + + # Assert that the snapshot has been deleted. + assert not subprocess.check_output( + f'python3 /app/tests/end-to-end/commands/fake_zfs.py list -H -t snapshot'.split(' ') + ) + finally: + shutil.rmtree(temporary_directory)