Add a "steps" option to run command hooks around particular sub-action steps like individual checks (#1134).

NEWS

@@ -1,8 +1,12 @@
 2.0.9.dev0
  * #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd
    service.
+ * #1134: Add a "steps" option to run command hooks around particular sub-action steps like
+   individual checks.
  * #1149: Add support for Python 3.14.
  * #1149: Include automated tests in the source dist tarball uploaded to PyPI.
+ * #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+
+   "slashdot" hack within source directory paths.
 
 2.0.8
  * #1114: Document systemd configuration changes for the ZFS filesystem hook.

borgmatic/actions/check.py

@@ -744,6 +744,7 @@ def run_check(
     global_arguments,
     local_path,
     remote_path,
+    hook_context,
 ):
     '''
     Run the "check" action for the given repository.
@@ -783,44 +784,75 @@ def run_check(
         archives_check_id,
     )
     borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
+    working_directory = borgmatic.config.paths.get_working_directory(config)
 
     if borg_specific_checks:
-        borgmatic.borg.check.check_archives(
-            repository['path'],
-            config,
-            local_borg_version,
-            check_arguments,
-            global_arguments,
-            borg_specific_checks,
-            archive_filter_flags,
-            local_path=local_path,
-            remote_path=remote_path,
-        )
-        for check in borg_specific_checks:
-            write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
+        with borgmatic.hooks.command.Before_after_hooks(
+            command_hooks=config.get('commands'),
+            before_after='step',
+            umask=config.get('umask'),
+            working_directory=working_directory,
+            dry_run=global_arguments.dry_run,
+            action_names=('check',),
+            step_names=('archives_repository_data',),
+            **hook_context,
+        ):
+            borgmatic.borg.check.check_archives(
+                repository['path'],
+                config,
+                local_borg_version,
+                check_arguments,
+                global_arguments,
+                borg_specific_checks,
+                archive_filter_flags,
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            for check in borg_specific_checks:
+                write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
 
     if 'extract' in checks:
-        borgmatic.borg.extract.extract_last_archive_dry_run(
-            config,
-            local_borg_version,
-            global_arguments,
-            repository['path'],
-            config.get('lock_wait'),
-            local_path,
-            remote_path,
-        )
-        write_check_time(make_check_time_path(config, repository_id, 'extract'))
-
-    if 'spot' in checks:
-        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
-            spot_check(
-                repository,
+        with borgmatic.hooks.command.Before_after_hooks(
+            command_hooks=config.get('commands'),
+            before_after='step',
+            umask=config.get('umask'),
+            working_directory=working_directory,
+            dry_run=global_arguments.dry_run,
+            action_names=('check',),
+            step_names=('extract',),
+            **hook_context,
+        ):
+            borgmatic.borg.extract.extract_last_archive_dry_run(
                 config,
                 local_borg_version,
                 global_arguments,
+                repository['path'],
+                config.get('lock_wait'),
                 local_path,
                 remote_path,
-                borgmatic_runtime_directory,
             )
+            write_check_time(make_check_time_path(config, repository_id, 'extract'))
 
-        write_check_time(make_check_time_path(config, repository_id, 'spot'))
+    if 'spot' in checks:
+        with borgmatic.hooks.command.Before_after_hooks(
+            command_hooks=config.get('commands'),
+            before_after='step',
+            umask=config.get('umask'),
+            working_directory=working_directory,
+            dry_run=global_arguments.dry_run,
+            action_names=('check',),
+            step_names=('spot',),
+            **hook_context,
+        ):
+            with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+                spot_check(
+                    repository,
+                    config,
+                    local_borg_version,
+                    global_arguments,
+                    local_path,
+                    remote_path,
+                    borgmatic_runtime_directory,
+                )
+
+            write_check_time(make_check_time_path(config, repository_id, 'spot'))

borgmatic/commands/borgmatic.py

@@ -463,6 +463,7 @@ def run_actions(  # noqa: PLR0912, PLR0915
                             global_arguments,
                             local_path,
                             remote_path,
+                            hook_context,
                         )
                 elif action_name == 'extract':
                     borgmatic.actions.extract.run_extract(

borgmatic/config/schema.yaml

@@ -1140,6 +1140,7 @@ properties:
                       before:
                           type: string
                           enum:
+                              - step
                               - action
                               - repository
                               - configuration
@@ -1148,6 +1149,8 @@ properties:
                               Name for the point in borgmatic's execution that
                               the commands should be run before (required if
                               "after" isn't set):
+                               * "step" runs before a sub-action step for each
+                              repository, e.g. for an individual check.
                                * "action" runs before each action for each
                               repository.
                                * "repository" runs before all actions for each
@@ -1188,6 +1191,18 @@ properties:
                               List of actions for which the commands will be
                               run. Defaults to running for all actions.
                           example: [create, prune, compact, check]
+                      steps:
+                          type: array
+                          items:
+                              type: string
+                              enum:
+                                  - archives_repository_data
+                                  - extract
+                                  - spot
+                          description: |
+                              List of sub-action steps for which the commands
+                              will be run. Defaults to running for all steps.
+                          example: [extract, spot]
                       run:
                           type: array
                           items:
@@ -1203,6 +1218,7 @@ properties:
                       after:
                           type: string
                           enum:
+                              - step
                               - action
                               - repository
                               - configuration
@@ -1212,6 +1228,8 @@ properties:
                               Name for the point in borgmatic's execution that
                               the commands should be run after (required if
                               "before" isn't set):
+                               * "step" runs before a sub-action step for each
+                              repository, e.g. for an individual check.
                                * "action" runs after each action for each
                               repository.
                                * "repository" runs after all actions for each
@@ -1254,6 +1272,18 @@ properties:
                               particular actions listed here. Defaults to
                               running for all actions.
                           example: [create, prune, compact, check]
+                      steps:
+                          type: array
+                          items:
+                              type: string
+                              enum:
+                                  - archives_repository_data
+                                  - extract
+                                  - spot
+                          description: |
+                              List of sub-action steps for which the commands
+                              will be run. Defaults to running for all steps.
+                          example: [extract, spot]
                       states:
                           type: array
                           items:

borgmatic/hooks/command.py

@@ -64,22 +64,27 @@ def make_environment(current_environment, sys_module=sys):
     return environment
 
 
-def filter_hooks(command_hooks, before=None, after=None, action_names=None, state_names=None):
+def filter_hooks(command_hooks, before=None, after=None, action_names=None, step_names=None, state_names=None):
     '''
     Given a sequence of command hook dicts from configuration and one or more filters (before name,
-    after name, a sequence of action names, and/or a sequence of execution result state names),
-    filter down the command hooks to just the ones that match the given filters.
+    after name, a sequence of action names, a sequence of sub-action steps, and/or a sequence of
+    execution result state names), filter down the command hooks to just the ones that match the
+    given filters.
     '''
     return tuple(
         hook_config
         for hook_config in command_hooks or ()
         for config_action_names in (hook_config.get('when'),)
+        for config_step_names in (hook_config.get('steps'),)
         for config_state_names in (hook_config.get('states'),)
         if before is None or hook_config.get('before') == before
         if after is None or hook_config.get('after') == after
         if action_names is None
         or config_action_names is None
         or set(config_action_names or ()).intersection(set(action_names))
+        if step_names is None
+        or config_step_names is None
+        or set(config_step_names or ()).intersection(set(step_names))
         if state_names is None
         or config_state_names is None
         or set(config_state_names or ()).intersection(set(state_names))
@@ -164,7 +169,8 @@ class Before_after_hooks:
            before_after='do_stuff',
            umask=config.get('umask'),
            dry_run=dry_run,
-           action_names=['create'],
+           action_names=['check'],
+           step_names=['spot'],
        ):
             do()
             some()
@@ -182,13 +188,14 @@ class Before_after_hooks:
         working_directory,
         dry_run,
         action_names=None,
+        step_names=None,
         **context,
     ):
         '''
         Given a sequence of command hook configuration dicts, the before/after name, a umask to run
         commands with, a working directory to run commands with, a dry run flag, a sequence of
-        action names, and any context for the executed commands, save those data points for use
-        below.
+        action names, a sequence of sub-action step names, and any context for the executed
+        commands, save those data points for use below.
         '''
         self.command_hooks = command_hooks
         self.before_after = before_after
@@ -196,6 +203,7 @@ class Before_after_hooks:
         self.working_directory = working_directory
         self.dry_run = dry_run
         self.action_names = action_names
+        self.step_names = step_names
         self.context = context
 
     def __enter__(self):
@@ -208,6 +216,7 @@ class Before_after_hooks:
                     self.command_hooks,
                     before=self.before_after,
                     action_names=self.action_names,
+                    step_names=self.step_names,
                 ),
                 self.umask,
                 self.working_directory,
@@ -234,6 +243,7 @@ class Before_after_hooks:
                     self.command_hooks,
                     after=self.before_after,
                     action_names=self.action_names,
+                    step_names=self.step_names,
                     state_names=['fail' if exception_type else 'finish'],
                 ),
                 self.umask,

borgmatic/hooks/data_source/btrfs.py

@@ -265,7 +265,10 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
     rewritten_path = initial_caret + os.path.join(
         subvolume_path,
         f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
-        '.',  # Borg 1.4+ "slashdot" hack.
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )

borgmatic/hooks/data_source/lvm.py

@@ -166,7 +166,10 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc
         hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
             MOUNT_POINT_HASH_LENGTH,
         ),
-        '.',  # Borg 1.4+ "slashdot" hack.
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )

borgmatic/hooks/data_source/zfs.py

@@ -214,7 +214,10 @@ def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory):
         # For instance, without this, snapshotting a dataset at /var and another at /var/spool would
         # result in overlapping snapshot patterns and therefore colliding mount attempts.
         hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
-        '.',  # Borg 1.4+ "slashdot" hack.
+        # Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
+        # included in the archive—but only if there's not already a slashdot hack present in the
+        # pattern.
+        ('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
         # Included so that the source directory ends up in the Borg archive at its "original" path.
         pattern.path.lstrip('^').lstrip(os.path.sep),
     )

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -77,12 +77,14 @@ commands:
 Each command in the `commands:` list has the following options:
 
  * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of:
+    * <span class="minilink minilink-addedin">New in version 2.0.9</span> `step` runs before or after each sub-action step for each repository, e.g. for an individual check.
     * `action` runs before or after each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc.
     * `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`.
     * `configuration` runs before or after all actions and repositories in the current configuration file.
     * `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`.
     * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook.
  * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions.
+ * `steps`: <span class="minilink minilink-addedin">New in version 2.0.9</span> Only trigger the hook when borgmatic runs particular sub-action steps (`extract`, `spot`, etc.) listed here. Defaults to running for all steps.
  * `states`: <span class="minilink minilink-addedin">New in version 2.0.3</span> Only trigger the hook if borgmatic encounters one of the states (execution results) listed here. This state is evaluated only for the scope of the configured `action`, `repository`, etc., rather than for the entire borgmatic run. Only available for `after` hooks. Defaults to running the hook for all states. One or more of:
     * `finish`: No errors occurred.
     * `fail`: An error occurred.
@@ -107,17 +109,22 @@ execution.
 
 Let's say you've got a borgmatic configuration file with a configured
 repository. And suppose you configure several command hooks and then run
-borgmatic for the `create` and `prune` actions. Here's the order of execution:
+borgmatic for the `create` and `check` actions. Here's the order of execution:
 
  * Run `before: everything` hooks (from all configuration files).
     * Run `before: configuration` hooks (from the first configuration file).
         * Run `before: repository` hooks (for the first repository).
             * Run `before: action` hooks for `create`.
-            * Actually run the `create` action (e.g. `borg create`).
+                * Run the `create` action including `borg create`.
             * Run `after: action` hooks for `create`.
-            * Run `before: action` hooks for `prune`.
-            * Actually run the `prune` action (e.g. `borg prune`).
-            * Run `after: action` hooks for `prune`.
+            * Run `before: action` hooks for `check`.
+                * Run `before: step` hooks for the `archives_repository_data` step.
+                    * Run the `borg check` portion of the `check` action.
+                * Run `after: step` hooks for the `archives_repository_data` step.
+                * Run `before: step` hooks for the `spot` step.
+                    * Run the `spot` check portion of the `check` action.
+                * Run `after: step` hooks for the `spot` step.
+            * Run `after: action` hooks for `check`.
         * Run `after: repository` hooks (for the first repository).
     * Run `after: configuration` hooks (from the first configuration file).
     * Run `after: error` hooks (if an error occurs).
@@ -134,9 +141,9 @@ have a chance to run. Whereas the `after: error` hook doesn't run until all
 actions for—and repositories in—a configuration file have had a chance to
 execute.
 
-And if there are multiple hooks defined for a particular step (e.g. `before:
-action` for `create`), then those hooks are run in the order they're defined in
-configuration.
+And if there are multiple hooks defined for a particular combination (e.g.
+`before: action` for `create`), then those hooks are run in the order they're
+defined in configuration.
 
 
 ### Deprecated command hooks

tests/unit/hooks/data_source/test_btrfs.py

@@ -326,6 +326,11 @@ def test_make_snapshot_path_includes_stripped_subvolume_path(
         ),
         ('/', Pattern('/foo'), Pattern('/.borgmatic-snapshot-1234/./foo')),
         ('/', Pattern('/'), Pattern('/.borgmatic-snapshot-1234/./')),
+        (
+            '/foo/bar',
+            Pattern('/foo/bar/./baz'),
+            Pattern('/foo/bar/.borgmatic-snapshot-1234/foo/bar/./baz'),
+        ),
     ),
 )
 def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(

tests/unit/hooks/data_source/test_lvm.py

@@ -270,6 +270,10 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
         ),
         (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')),
         (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')),
+        (
+            Pattern('/foo/./bar/baz'),
+            Pattern('/run/borgmatic/lvm_snapshots/b33f/foo/./bar/baz'),
+        ),
     ),
 )
 def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(

tests/unit/hooks/data_source/test_zfs.py

@@ -278,6 +278,10 @@ def test_get_all_dataset_mount_points_omits_duplicates():
         ),
         (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')),
         (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')),
+        (
+            Pattern('/foo/./bar/baz'),
+            Pattern('/run/borgmatic/zfs_snapshots/b33f/foo/./bar/baz'),
+        ),
     ),
 )
 def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(