diff --git a/NEWS b/NEWS index dc5fe202..3bef5b2d 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,12 @@ 2.0.9.dev0 * #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd service. + * #1134: Add a "steps" option to run command hooks around particular sub-action steps like + individual checks. * #1149: Add support for Python 3.14. * #1149: Include automated tests in the source dist tarball uploaded to PyPI. + * #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+ + "slashdot" hack within source directory paths. 2.0.8 * #1114: Document systemd configuration changes for the ZFS filesystem hook. diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 0872d528..09e03085 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -744,6 +744,7 @@ def run_check( global_arguments, local_path, remote_path, + hook_context, ): ''' Run the "check" action for the given repository. @@ -783,44 +784,75 @@ def run_check( archives_check_id, ) borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'}) + working_directory = borgmatic.config.paths.get_working_directory(config) if borg_specific_checks: - borgmatic.borg.check.check_archives( - repository['path'], - config, - local_borg_version, - check_arguments, - global_arguments, - borg_specific_checks, - archive_filter_flags, - local_path=local_path, - remote_path=remote_path, - ) - for check in borg_specific_checks: - write_check_time(make_check_time_path(config, repository_id, check, archives_check_id)) + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='step', + umask=config.get('umask'), + working_directory=working_directory, + dry_run=global_arguments.dry_run, + action_names=('check',), + step_names=('archives_repository_data',), + **hook_context, + ): + borgmatic.borg.check.check_archives( + repository['path'], + config, + local_borg_version, + check_arguments, + global_arguments, + borg_specific_checks, + archive_filter_flags, + local_path=local_path, + remote_path=remote_path, + ) + for check in borg_specific_checks: + write_check_time(make_check_time_path(config, repository_id, check, archives_check_id)) if 'extract' in checks: - borgmatic.borg.extract.extract_last_archive_dry_run( - config, - local_borg_version, - global_arguments, - repository['path'], - config.get('lock_wait'), - local_path, - remote_path, - ) - write_check_time(make_check_time_path(config, repository_id, 'extract')) - - if 'spot' in checks: - with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory: - spot_check( - repository, + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='step', + umask=config.get('umask'), + working_directory=working_directory, + dry_run=global_arguments.dry_run, + action_names=('check',), + step_names=('extract',), + **hook_context, + ): + borgmatic.borg.extract.extract_last_archive_dry_run( config, local_borg_version, global_arguments, + repository['path'], + config.get('lock_wait'), local_path, remote_path, - borgmatic_runtime_directory, ) + write_check_time(make_check_time_path(config, repository_id, 'extract')) - write_check_time(make_check_time_path(config, repository_id, 'spot')) + if 'spot' in checks: + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='step', + umask=config.get('umask'), + working_directory=working_directory, + dry_run=global_arguments.dry_run, + action_names=('check',), + step_names=('spot',), + **hook_context, + ): + with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory: + spot_check( + repository, + config, + local_borg_version, + global_arguments, + local_path, + remote_path, + borgmatic_runtime_directory, + ) + + write_check_time(make_check_time_path(config, repository_id, 'spot')) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 768d339e..cb0c0dd1 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -463,6 +463,7 @@ def run_actions( # noqa: PLR0912, PLR0915 global_arguments, local_path, remote_path, + hook_context, ) elif action_name == 'extract': borgmatic.actions.extract.run_extract( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f65b2f96..f7cf2cba 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1140,6 +1140,7 @@ properties: before: type: string enum: + - step - action - repository - configuration @@ -1148,6 +1149,8 @@ properties: Name for the point in borgmatic's execution that the commands should be run before (required if "after" isn't set): + * "step" runs before a sub-action step for each + repository, e.g. for an individual check. * "action" runs before each action for each repository. * "repository" runs before all actions for each @@ -1188,6 +1191,18 @@ properties: List of actions for which the commands will be run. Defaults to running for all actions. example: [create, prune, compact, check] + steps: + type: array + items: + type: string + enum: + - archives_repository_data + - extract + - spot + description: | + List of sub-action steps for which the commands + will be run. Defaults to running for all steps. + example: [extract, spot] run: type: array items: @@ -1203,6 +1218,7 @@ properties: after: type: string enum: + - step - action - repository - configuration @@ -1212,6 +1228,8 @@ properties: Name for the point in borgmatic's execution that the commands should be run after (required if "before" isn't set): + * "step" runs before a sub-action step for each + repository, e.g. for an individual check. * "action" runs after each action for each repository. * "repository" runs after all actions for each @@ -1254,6 +1272,18 @@ properties: particular actions listed here. Defaults to running for all actions. example: [create, prune, compact, check] + steps: + type: array + items: + type: string + enum: + - archives_repository_data + - extract + - spot + description: | + List of sub-action steps for which the commands + will be run. Defaults to running for all steps. + example: [extract, spot] states: type: array items: diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 6140d0bc..591624a4 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -64,22 +64,27 @@ def make_environment(current_environment, sys_module=sys): return environment -def filter_hooks(command_hooks, before=None, after=None, action_names=None, state_names=None): +def filter_hooks(command_hooks, before=None, after=None, action_names=None, step_names=None, state_names=None): ''' Given a sequence of command hook dicts from configuration and one or more filters (before name, - after name, a sequence of action names, and/or a sequence of execution result state names), - filter down the command hooks to just the ones that match the given filters. + after name, a sequence of action names, a sequence of sub-action steps, and/or a sequence of + execution result state names), filter down the command hooks to just the ones that match the + given filters. ''' return tuple( hook_config for hook_config in command_hooks or () for config_action_names in (hook_config.get('when'),) + for config_step_names in (hook_config.get('steps'),) for config_state_names in (hook_config.get('states'),) if before is None or hook_config.get('before') == before if after is None or hook_config.get('after') == after if action_names is None or config_action_names is None or set(config_action_names or ()).intersection(set(action_names)) + if step_names is None + or config_step_names is None + or set(config_step_names or ()).intersection(set(step_names)) if state_names is None or config_state_names is None or set(config_state_names or ()).intersection(set(state_names)) @@ -164,7 +169,8 @@ class Before_after_hooks: before_after='do_stuff', umask=config.get('umask'), dry_run=dry_run, - action_names=['create'], + action_names=['check'], + step_names=['spot'], ): do() some() @@ -182,13 +188,14 @@ class Before_after_hooks: working_directory, dry_run, action_names=None, + step_names=None, **context, ): ''' Given a sequence of command hook configuration dicts, the before/after name, a umask to run commands with, a working directory to run commands with, a dry run flag, a sequence of - action names, and any context for the executed commands, save those data points for use - below. + action names, a sequence of sub-action step names, and any context for the executed + commands, save those data points for use below. ''' self.command_hooks = command_hooks self.before_after = before_after @@ -196,6 +203,7 @@ class Before_after_hooks: self.working_directory = working_directory self.dry_run = dry_run self.action_names = action_names + self.step_names = step_names self.context = context def __enter__(self): @@ -208,6 +216,7 @@ class Before_after_hooks: self.command_hooks, before=self.before_after, action_names=self.action_names, + step_names=self.step_names, ), self.umask, self.working_directory, @@ -234,6 +243,7 @@ class Before_after_hooks: self.command_hooks, after=self.before_after, action_names=self.action_names, + step_names=self.step_names, state_names=['fail' if exception_type else 'finish'], ), self.umask, diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 85adb245..3bfe700b 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -265,7 +265,10 @@ def make_borg_snapshot_pattern(subvolume_path, pattern): rewritten_path = initial_caret + os.path.join( subvolume_path, f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}', - '.', # Borg 1.4+ "slashdot" hack. + # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting + # included in the archive—but only if there's not already a slashdot hack present in the + # pattern. + ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'), # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), ) diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 1e5fd266..1d43d224 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -166,7 +166,10 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( MOUNT_POINT_HASH_LENGTH, ), - '.', # Borg 1.4+ "slashdot" hack. + # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting + # included in the archive—but only if there's not already a slashdot hack present in the + # pattern. + ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'), # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), ) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index a9745483..988428c0 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -214,7 +214,10 @@ def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory): # For instance, without this, snapshotting a dataset at /var and another at /var/spool would # result in overlapping snapshot patterns and therefore colliding mount attempts. hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), - '.', # Borg 1.4+ "slashdot" hack. + # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting + # included in the archive—but only if there's not already a slashdot hack present in the + # pattern. + ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'), # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), ) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index e2dd7361..2ae97996 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -77,12 +77,14 @@ commands: Each command in the `commands:` list has the following options: * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of: + * New in version 2.0.9 `step` runs before or after each sub-action step for each repository, e.g. for an individual check. * `action` runs before or after each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc. * `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`. * `configuration` runs before or after all actions and repositories in the current configuration file. * `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`. * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook. * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. + * `steps`: New in version 2.0.9 Only trigger the hook when borgmatic runs particular sub-action steps (`extract`, `spot`, etc.) listed here. Defaults to running for all steps. * `states`: New in version 2.0.3 Only trigger the hook if borgmatic encounters one of the states (execution results) listed here. This state is evaluated only for the scope of the configured `action`, `repository`, etc., rather than for the entire borgmatic run. Only available for `after` hooks. Defaults to running the hook for all states. One or more of: * `finish`: No errors occurred. * `fail`: An error occurred. @@ -107,17 +109,22 @@ execution. Let's say you've got a borgmatic configuration file with a configured repository. And suppose you configure several command hooks and then run -borgmatic for the `create` and `prune` actions. Here's the order of execution: +borgmatic for the `create` and `check` actions. Here's the order of execution: * Run `before: everything` hooks (from all configuration files). * Run `before: configuration` hooks (from the first configuration file). * Run `before: repository` hooks (for the first repository). * Run `before: action` hooks for `create`. - * Actually run the `create` action (e.g. `borg create`). + * Run the `create` action including `borg create`. * Run `after: action` hooks for `create`. - * Run `before: action` hooks for `prune`. - * Actually run the `prune` action (e.g. `borg prune`). - * Run `after: action` hooks for `prune`. + * Run `before: action` hooks for `check`. + * Run `before: step` hooks for the `archives_repository_data` step. + * Run the `borg check` portion of the `check` action. + * Run `after: step` hooks for the `archives_repository_data` step. + * Run `before: step` hooks for the `spot` step. + * Run the `spot` check portion of the `check` action. + * Run `after: step` hooks for the `spot` step. + * Run `after: action` hooks for `check`. * Run `after: repository` hooks (for the first repository). * Run `after: configuration` hooks (from the first configuration file). * Run `after: error` hooks (if an error occurs). @@ -134,9 +141,9 @@ have a chance to run. Whereas the `after: error` hook doesn't run until all actions for—and repositories in—a configuration file have had a chance to execute. -And if there are multiple hooks defined for a particular step (e.g. `before: -action` for `create`), then those hooks are run in the order they're defined in -configuration. +And if there are multiple hooks defined for a particular combination (e.g. +`before: action` for `create`), then those hooks are run in the order they're +defined in configuration. ### Deprecated command hooks diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 0ad67d0f..81a0cc42 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -326,6 +326,11 @@ def test_make_snapshot_path_includes_stripped_subvolume_path( ), ('/', Pattern('/foo'), Pattern('/.borgmatic-snapshot-1234/./foo')), ('/', Pattern('/'), Pattern('/.borgmatic-snapshot-1234/./')), + ( + '/foo/bar', + Pattern('/foo/bar/./baz'), + Pattern('/foo/bar/.borgmatic-snapshot-1234/foo/bar/./baz'), + ), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index e692a48c..50ada6f5 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -270,6 +270,10 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ), (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')), (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')), + ( + Pattern('/foo/./bar/baz'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/foo/./bar/baz'), + ), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 5d51213a..2af87e3c 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -278,6 +278,10 @@ def test_get_all_dataset_mount_points_omits_duplicates(): ), (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')), (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')), + ( + Pattern('/foo/./bar/baz'), + Pattern('/run/borgmatic/zfs_snapshots/b33f/foo/./bar/baz'), + ), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(