Add an end-to-end test for the ZFS hook using a fake ZFS binary (#80).

borgmatic/hooks/data_source/btrfs.py

@@ -24,8 +24,8 @@ def get_filesystem_mount_points(findmnt_command):
     Given a findmnt command to run, get all top-level Btrfs filesystem mount points.
     '''
     findmnt_output = borgmatic.execute.execute_command_and_capture_output(
-        (
-            findmnt_command,
+        tuple(findmnt_command.split(' '))
+        + (
             '-nt',
             'btrfs',
         )
@@ -40,8 +40,8 @@ def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point):
     that filesystem. Include the filesystem itself.
     '''
     btrfs_output = borgmatic.execute.execute_command_and_capture_output(
-        (
-            btrfs_command,
+        tuple(btrfs_command.split(' '))
+        + (
             'subvolume',
             'list',
             filesystem_mount_point,
@@ -167,8 +167,8 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma:
     os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        (
-            btrfs_command,
+        tuple(btrfs_command.split(' '))
+        + (
             'subvolume',
             'snapshot',
             '-r',  # Read-only,
@@ -242,8 +242,8 @@ def delete_snapshot(btrfs_command, snapshot_path):  # pragma: no cover
     Given a Btrfs command to run and the name of a snapshot path, delete it.
     '''
     borgmatic.execute.execute_command(
-        (
-            btrfs_command,
+        tuple(btrfs_command.split(' '))
+        + (
             'subvolume',
             'delete',
             snapshot_path,

borgmatic/hooks/data_source/lvm.py

@@ -51,7 +51,7 @@ def get_logical_volumes(lsblk_command, source_directories=None):
             )
         )
     except json.JSONDecodeError as error:
-        raise ValueError('Invalid {lsblk_command} JSON output: {error}')
+        raise ValueError(f'Invalid {lsblk_command} JSON output: {error}')
 
     candidate_source_directories = set(source_directories or ())
 
@@ -84,8 +84,8 @@ def snapshot_logical_volume(
     snapshot, and a snapshot size string, create a new LVM snapshot.
     '''
     borgmatic.execute.execute_command(
-        (
-            lvcreate_command,
+        tuple(lvcreate_command.split(' '))
+        + (
             '--snapshot',
             ('--extents' if '%' in snapshot_size else '--size'),
             snapshot_size,
@@ -106,8 +106,8 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path):  # prag
     os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        (
-            mount_command,
+        tuple(mount_command.split(' '))
+        + (
             '-o',
             'ro',
             snapshot_device,
@@ -221,8 +221,8 @@ def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
     Given a umount command to run and the mount path of a snapshot, unmount it.
     '''
     borgmatic.execute.execute_command(
-        (
-            umount_command,
+        tuple(umount_command.split(' '))
+        + (
             snapshot_mount_path,
         ),
         output_log_level=logging.DEBUG,
@@ -234,8 +234,8 @@ def delete_snapshot(lvremove_command, snapshot_device_path):  # pragma: no cover
     Given an lvremove command to run and the device path of a snapshot, remove it it.
     '''
     borgmatic.execute.execute_command(
-        (
-            lvremove_command,
+        tuple(lvremove_command.split(' '))
+        + (
             '--force',  # Suppress an interactive "are you sure?" type prompt.
             snapshot_device_path,
         ),
@@ -258,9 +258,9 @@ def get_snapshots(lvs_command, snapshot_name=None):
     try:
         snapshot_info = json.loads(
             borgmatic.execute.execute_command_and_capture_output(
-                (
-                    # Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
-                    lvs_command,
+                # Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
+                tuple(lvs_command.split(' '))
+                + (
                     '--report-format',
                     'json',
                     '--options',
@@ -271,7 +271,7 @@ def get_snapshots(lvs_command, snapshot_name=None):
             )
         )
     except json.JSONDecodeError as error:
-        raise ValueError('Invalid {lvs_command} JSON output: {error}')
+        raise ValueError(f'Invalid {lvs_command} JSON output: {error}')
 
     try:
         return tuple(

borgmatic/hooks/data_source/zfs.py

@@ -41,8 +41,8 @@ def get_datasets_to_backup(zfs_command, source_directories):
     Return the result as a sequence of Dataset instances, sorted by mount point.
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
-        (
-            zfs_command,
+        tuple(zfs_command.split(' '))
+        + (
             'list',
             '-H',
             '-t',
@@ -67,7 +67,7 @@ def get_datasets_to_backup(zfs_command, source_directories):
             reverse=True,
         )
     except ValueError:
-        raise ValueError('Invalid {zfs_command} list output')
+        raise ValueError(f'Invalid {zfs_command} list output')
 
     candidate_source_directories = set(source_directories)
 
@@ -102,8 +102,8 @@ def get_all_dataset_mount_points(zfs_command):
     Given a ZFS command to run, return all ZFS datasets as a sequence of sorted mount points.
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
-        (
-            zfs_command,
+        tuple(zfs_command.split(' '))
+        + (
             'list',
             '-H',
             '-t',
@@ -122,8 +122,8 @@ def snapshot_dataset(zfs_command, full_snapshot_name):  # pragma: no cover
     snapshot.
     '''
     borgmatic.execute.execute_command(
-        (
-            zfs_command,
+        tuple(zfs_command.split(' '))
+        + (
             'snapshot',
             full_snapshot_name,
         ),
@@ -140,8 +140,8 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
     os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
 
     borgmatic.execute.execute_command(
-        (
-            mount_command,
+        tuple(mount_command.split(' '))
+        + (
             '-t',
             'zfs',
             full_snapshot_name,
@@ -237,8 +237,8 @@ def unmount_snapshot(umount_command, snapshot_mount_path):  # pragma: no cover
     Given a umount command to run and the mount path of a snapshot, unmount it.
     '''
     borgmatic.execute.execute_command(
-        (
-            umount_command,
+        tuple(umount_command.split(' '))
+        + (
             snapshot_mount_path,
         ),
         output_log_level=logging.DEBUG,
@@ -251,8 +251,8 @@ def destroy_snapshot(zfs_command, full_snapshot_name):  # pragma: no cover
     it.
     '''
     borgmatic.execute.execute_command(
-        (
-            zfs_command,
+        tuple(zfs_command.split(' '))
+        + (
             'destroy',
             full_snapshot_name,
         ),
@@ -266,8 +266,8 @@ def get_all_snapshots(zfs_command):
     form "dataset@snapshot".
     '''
     list_output = borgmatic.execute.execute_command_and_capture_output(
-        (
-            zfs_command,
+        tuple(zfs_command.split(' '))
+        + (
             'list',
             '-H',
             '-t',

tests/end-to-end/commands/fake_mount.py

@@ -0,0 +1,26 @@
+import argparse
+import os
+import sys
+
+
+def parse_arguments(*unparsed_arguments):
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument('-t', dest='type')
+    parser.add_argument('snapshot_name')
+    parser.add_argument('mount_point')
+
+    return parser.parse_args(unparsed_arguments)
+
+
+def main():
+    arguments = parse_arguments(*sys.argv[1:])
+
+    subdirectory = os.path.join(arguments.mount_point, 'subdir')
+    os.mkdir(subdirectory)
+    test_file = open(os.path.join(subdirectory, 'file.txt'), 'w')
+    test_file.write('contents')
+    test_file.close()
+
+
+if __name__ == '__main__':
+    main()

tests/end-to-end/commands/fake_umount.py

@@ -0,0 +1,22 @@
+import argparse
+import os
+import shutil
+import sys
+
+
+def parse_arguments(*unparsed_arguments):
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument('mount_point')
+
+    return parser.parse_args(unparsed_arguments)
+
+
+def main():
+    arguments = parse_arguments(*sys.argv[1:])
+
+    subdirectory = os.path.join(arguments.mount_point, 'subdir')
+    shutil.rmtree(subdirectory)
+
+
+if __name__ == '__main__':
+    main()

tests/end-to-end/commands/fake_zfs.py

@@ -0,0 +1,103 @@
+import argparse
+import json
+import sys
+
+
+def parse_arguments(*unparsed_arguments):
+    global_parser = argparse.ArgumentParser(add_help=False)
+    action_parsers = global_parser.add_subparsers(dest='action')
+
+    list_parser = action_parsers.add_parser('list')
+    list_parser.add_argument('-H', dest='header', action='store_false', default=True)
+    list_parser.add_argument('-t', dest='type', default='filesystem')
+    list_parser.add_argument('-o', dest='properties', default='name,used,avail,refer,mountpoint')
+
+    snapshot_parser = action_parsers.add_parser('snapshot')
+    snapshot_parser.add_argument('name')
+
+    destroy_parser = action_parsers.add_parser('destroy')
+    destroy_parser.add_argument('name')
+
+    return global_parser.parse_args(unparsed_arguments)
+
+
+BUILTIN_DATASETS = (
+    {
+        'name': 'pool',
+        'used': '256K',
+        'avail': '23.7M',
+        'refer': '25K',
+        'mountpoint': '/pool',
+    },
+    {
+        'name': 'pool/dataset',
+        'used': '256K',
+        'avail': '23.7M',
+        'refer': '25K',
+        'mountpoint': '/pool/dataset',
+    },
+)
+
+
+
+
+
+def load_snapshots():
+    try:
+        return json.load(open('/tmp/fake_zfs.json'))
+    except FileNotFoundError:
+        return []
+
+
+def save_snapshots(snapshots):
+    json.dump(snapshots, open('/tmp/fake_zfs.json', 'w'))
+
+
+def print_dataset_list(arguments, datasets, snapshots):
+    properties = arguments.properties.split(',')
+    data = (
+        (
+            tuple(property_name.upper() for property_name in properties),
+        )
+        if arguments.header else ()
+    ) + tuple(
+        tuple(
+            dataset.get(property_name, '-') for property_name in properties
+        )
+        for dataset in (snapshots if arguments.type == 'snapshot' else datasets)
+    )
+
+    if not data:
+        return
+
+    for data_row in data:
+        print('\t'.join(data_row))
+
+
+def main():
+    arguments = parse_arguments(*sys.argv[1:])
+    snapshots = load_snapshots()
+
+    if arguments.action == 'list':
+        print_dataset_list(arguments, BUILTIN_DATASETS, snapshots)
+    elif arguments.action == 'snapshot':
+        snapshots.append(
+            {
+                'name': arguments.name,
+                'used': '0B',
+                'avail': '-',
+                'refer': '25K',
+                'mountpoint': '-',
+            },
+        )
+        save_snapshots(snapshots)
+    elif arguments.action == 'destroy':
+        snapshots = [
+            snapshot for snapshot in snapshots
+            if snapshot['name'] != arguments.name
+        ]
+        save_snapshots(snapshots)
+
+
+if __name__ == '__main__':
+    main()

tests/end-to-end/test_zfs.py

@@ -0,0 +1,62 @@
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def generate_configuration(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing (including injecting the given repository path and tacking on an encryption
+    passphrase).
+    '''
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
+    config = (
+        open(config_path)
+        .read()
+        .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path)
+        .replace('- path: /mnt/backup', '')
+        .replace('label: local', '')
+        .replace('- /home', f'- {config_path}')
+        .replace('- /etc', '- /pool/dataset')
+        .replace('- /var/log/syslog*', '')
+        + 'encryption_passphrase: "test"\n'
+        + 'zfs:\n'
+        + '    zfs_command: python3 /app/tests/end-to-end/commands/fake_zfs.py\n'
+        + '    mount_command: python3 /app/tests/end-to-end/commands/fake_mount.py\n'
+        + '    umount_command: python3 /app/tests/end-to-end/commands/fake_umount.py'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_zfs_create_and_list():
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path)
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' ')
+        )
+
+        # Run a create action to exercise ZFS snapshotting and backup.
+        subprocess.check_call(f'borgmatic --config {config_path} create'.split(' '))
+
+        # List the resulting archive and assert that the snapshotted files are there.
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} list --archive latest'.split(' ')
+        ).decode(sys.stdout.encoding)
+
+        assert 'pool/dataset/subdir/file.txt' in output
+
+        # Assert that the snapshot has been deleted.
+        assert not subprocess.check_output(
+            f'python3 /app/tests/end-to-end/commands/fake_zfs.py list -H -t snapshot'.split(' ')
+        )
+    finally:
+        shutil.rmtree(temporary_directory)