From e69c686abf65a4f46fd85b2a1a0cd43a26941243 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 11:32:57 -0800 Subject: [PATCH] Get all unit/integration tests passing (#1001). --- borgmatic/hooks/data_source/lvm.py | 8 +- borgmatic/hooks/data_source/snapshot.py | 2 - tests/unit/actions/test_create.py | 31 +- tests/unit/borg/test_create.py | 4 +- tests/unit/hooks/data_source/test_btrfs.py | 14 +- tests/unit/hooks/data_source/test_lvm.py | 408 ++++++++++++++------- tests/unit/hooks/data_source/test_zfs.py | 116 +++--- 7 files changed, 371 insertions(+), 212 deletions(-) diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 1d2e4b42..186234f3 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -138,7 +138,7 @@ MOUNT_POINT_HASH_LENGTH = 10 def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory): ''' Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume - contianing it, return a new Pattern with its path rewritten to be in a snapshot directory based + containing it, return a new Pattern with its path rewritten to be in a snapshot directory based on both the given runtime directory and the given Logical_volume's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break @@ -158,7 +158,9 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc # volumes. For instance, without this, snapshotting a logical volume at /var and another at # /var/spool would result in overlapping snapshot patterns and therefore colliding mount # attempts. - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -237,7 +239,7 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'lvm_snapshots', - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( MOUNT_POINT_HASH_LENGTH ), logical_volume.mount_point.lstrip(os.path.sep), diff --git a/borgmatic/hooks/data_source/snapshot.py b/borgmatic/hooks/data_source/snapshot.py index b13f1aa3..ece63ce6 100644 --- a/borgmatic/hooks/data_source/snapshot.py +++ b/borgmatic/hooks/data_source/snapshot.py @@ -1,5 +1,3 @@ -import logging - import pathlib IS_A_HOOK = False diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 8ba1eed0..5e4a4c9f 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -5,16 +5,21 @@ import pytest from flexmock import flexmock from borgmatic.actions import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type @pytest.mark.parametrize( 'pattern_line,expected_pattern', ( - ('R /foo', Pattern('/foo')), - ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)), - ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)), - ('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)), + ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)), + ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)), + ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)), + ( + '+ sh:/foo*', + Pattern( + '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), + ), ), ) def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern): @@ -28,8 +33,8 @@ def test_parse_pattern_with_invalid_pattern_line_errors(): def test_collect_patterns_converts_source_directories(): assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == ( - Pattern('/foo'), - Pattern('/bar'), + Pattern('/foo', source=Pattern_source.CONFIG), + Pattern('/bar', source=Pattern_source.CONFIG), ) @@ -48,9 +53,15 @@ def test_collect_patterns_parses_config_patterns(): def test_collect_patterns_converts_exclude_patterns(): assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == ( - Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL), + Pattern( + '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 6015004a..be2ceede 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -4,7 +4,7 @@ import pytest from flexmock import flexmock from borgmatic.borg import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from ..test_verbosity import insert_logging_mock @@ -663,6 +663,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', @@ -713,6 +714,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 65c4ccbe..648902c3 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import btrfs as module @@ -54,8 +54,8 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns(): contained_pattern = Pattern( '/mnt1', - type=module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + type=Pattern_type.ROOT, + source=Pattern_source.CONFIG, ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -83,8 +83,8 @@ def test_get_subvolumes_skips_non_root_patterns(): ( Pattern( '/mnt1', - type=module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + type=Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, ), ) ) @@ -114,8 +114,8 @@ def test_get_subvolumes_skips_non_config_patterns(): ( Pattern( '/mnt1', - type=module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.HOOK, + type=Pattern_type.ROOT, + source=Pattern_source.HOOK, ), ) ) diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 49159905..9e172de2 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import lvm as module @@ -37,14 +37,20 @@ def test_get_logical_volumes_filters_by_patterns(): } ''' ) - contained = {Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')} + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + } flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args(None, contained).never() flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/mnt/lvolume', contained).and_return( - (Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + ( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ) ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -54,17 +60,116 @@ def test_get_logical_volumes_filters_by_patterns(): ).with_args('/mnt/notlvm', contained).never() assert module.get_logical_volumes( - 'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ) == ( module.Logical_volume( name='vgroup-lvolume', device_path='/dev/mapper/vgroup-lvolume', mount_point='/mnt/lvolume', - contained_patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')), + contained_patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ), ) +def test_get_logical_volumes_skips_non_root_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern( + '/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG + ), + ), + ) + == () + ) + + +def test_get_logical_volumes_skips_non_config_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ), + ) + == () + ) + + def test_get_logical_volumes_with_invalid_lsblk_json_errors(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -138,13 +243,13 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( ( Pattern('/foo/bar/baz'), - Pattern('/run/borgmatic/lvm_snapshots/./foo/bar/baz'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar/baz'), ), - (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/./foo/bar')), + (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar')), ( Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '^/run/borgmatic/lvm_snapshots/./foo/bar', + '^/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), @@ -152,40 +257,48 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '/run/borgmatic/lvm_snapshots/./foo/bar', + '/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), ), - (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/./foo')), - (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/./')), + (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')), + (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( pattern, expected_pattern ): - assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + + assert ( + module.make_borg_snapshot_pattern( + pattern, flexmock(mount_point='/something'), '/run/borgmatic' + ) + == expected_pattern + ) def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -203,18 +316,21 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -229,8 +345,8 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -259,22 +375,21 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): def test_dump_data_sources_uses_snapshot_size_for_snapshot(): config = {'lvm': {'snapshot_size': '1000PB'}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', @@ -298,18 +413,21 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -324,8 +442,8 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -339,22 +457,21 @@ def test_dump_data_sources_uses_custom_commands(): }, } patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( '/usr/local/bin/lvcreate', @@ -378,18 +495,25 @@ def test_dump_data_sources_uses_custom_commands(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/usr/local/bin/mount', + '/dev/lvolume1_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/usr/local/bin/mount', + '/dev/lvolume2_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -404,8 +528,8 @@ def test_dump_data_sources_uses_custom_commands(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -463,22 +587,21 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): config = {'lvm': {}} patterns = [Pattern('/hmm')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -496,18 +619,21 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -523,8 +649,8 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained assert patterns == [ Pattern('/hmm'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -694,16 +820,18 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -799,9 +927,11 @@ def test_remove_data_source_dumps_with_missing_snapshot_directory_skips_unmount( flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(False) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() @@ -843,24 +973,26 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -900,24 +1032,26 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(True).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True).and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -957,16 +1091,18 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(FileNotFoundError) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).never() flexmock(module).should_receive('get_snapshots').never() flexmock(module).should_receive('remove_snapshot').never() @@ -1000,16 +1136,18 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(module.subprocess.CalledProcessError(1, 'wtf')) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).never() flexmock(module).should_receive('get_snapshots').never() flexmock(module).should_receive('remove_snapshot').never() @@ -1043,16 +1181,18 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise(FileNotFoundError) flexmock(module).should_receive('remove_snapshot').never() @@ -1086,16 +1226,18 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise( module.subprocess.CalledProcessError(1, 'wtf') diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 7bc07122..5900e037 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -3,7 +3,7 @@ import os import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import zfs as module @@ -19,8 +19,8 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): ( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), ) ) @@ -33,18 +33,18 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): patterns=( Pattern( '/foo', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), Pattern( '/bar', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), ), ) == ( @@ -54,8 +54,8 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): contained_patterns=( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), ), ), @@ -74,8 +74,8 @@ def test_get_datasets_to_backup_skips_non_root_patterns(): ( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, ), ) ) @@ -83,26 +83,29 @@ def test_get_datasets_to_backup_skips_non_root_patterns(): 'get_contained_patterns' ).with_args('/other', object).and_return(()) - assert module.get_datasets_to_backup( - 'zfs', - patterns=( - Pattern( - '/foo', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), ), - Pattern( - '/dataset', - module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, - ), - Pattern( - '/bar', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, - ), - ), - ) == () + ) + == () + ) def test_get_datasets_to_backup_skips_non_config_patterns(): @@ -117,8 +120,8 @@ def test_get_datasets_to_backup_skips_non_config_patterns(): ( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.HOOK, + Pattern_type.ROOT, + source=Pattern_source.HOOK, ), ) ) @@ -126,26 +129,29 @@ def test_get_datasets_to_backup_skips_non_config_patterns(): 'get_contained_patterns' ).with_args('/other', object).and_return(()) - assert module.get_datasets_to_backup( - 'zfs', - patterns=( - Pattern( - '/foo', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), ), - Pattern( - '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.HOOK, - ), - Pattern( - '/bar', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, - ), - ), - ) == () + ) + == () + ) def test_get_datasets_to_backup_filters_datasets_by_user_property(): @@ -169,9 +175,7 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property(): name='dataset', mount_point='/dataset', auto_backup=True, - contained_patterns=( - Pattern('/dataset', source=module.borgmatic.borg.pattern.Pattern_source.HOOK), - ), + contained_patterns=(Pattern('/dataset', source=Pattern_source.HOOK),), ), )