From 8817364e6d4083aa570add693142909d6b63c193 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Mar 2025 22:38:48 -0700 Subject: [PATCH] Documentation on command hooks (#1019). --- NEWS | 2 +- borgmatic/commands/borgmatic.py | 8 +- borgmatic/config/schema.yaml | 9 +- docs/_includes/index.css | 1 + ...reparation-and-cleanup-steps-to-backups.md | 153 ++++++++++----- docs/how-to/monitor-your-backups.md | 181 +++++------------- 6 files changed, 164 insertions(+), 190 deletions(-) diff --git a/NEWS b/NEWS index bff2f68a..a3a2852f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -2.0.0dev0 +2.0.0.dev0 * #790: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index eea3793a..102ceef2 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -85,7 +85,7 @@ def run_configuration(config_filename, config, config_paths, arguments): retries = config.get('retries', 0) retry_wait = config.get('retry_wait', 0) encountered_error = None - error_repository = '' + error_repository = None using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED @@ -192,7 +192,7 @@ def run_configuration(config_filename, config, config_paths, arguments): error, ) encountered_error = error - error_repository = repository['path'] + error_repository = repository try: if monitoring_hooks_are_activated: @@ -246,7 +246,9 @@ def run_configuration(config_filename, config, config_paths, arguments): config.get('umask'), global_arguments.dry_run, configuration_filename=config_filename, - repository=error_repository, + log_file=arguments['global'].log_file, + repository=error_repository.get('path', '') if error_repository else '', + repository_label=error_repository.get('label', '') if error_repository else '', error=encountered_error, output=getattr(encountered_error, 'output', ''), ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 31129af2..13ab0686 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1091,10 +1091,11 @@ properties: - key - borg description: | - List of actions for which the commands will be - run. Defaults to running for all actions. Ignored - for "dump_data_sources", which by its nature only - runs for "create". + Only trigger the hook when borgmatic is run with + particular actions listed here. Defaults to + running for all actions. Ignored for + "dump_data_sources", which by its nature only runs + for "create". example: [create, prune, compact, check] run: type: array diff --git a/docs/_includes/index.css b/docs/_includes/index.css index 56841ecb..c798f2e9 100644 --- a/docs/_includes/index.css +++ b/docs/_includes/index.css @@ -165,6 +165,7 @@ ul { } li { padding: .25em 0; + line-height: 1.5; } li ul { list-style-type: disc; diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 98f20ccf..33a6593a 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -33,6 +33,9 @@ commands: - prune run: - echo "After create and/or prune!" + - after: error + run: + - echo "Something went wrong!" ``` If a `run:` command contains a special YAML character such as a colon, you may @@ -55,9 +58,12 @@ Each command in the `commands:` list has the following options: * `configuration` runs before or after all actions and repositories in the current configuration file. * `everything` runs before or after all configuration files. This replaces the deprecated `before_everything` and `after_everything`. * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook. - * `when`: Actions (`create`, `prune`, etc.) for which the commands will be run. Defaults to running for all actions. + * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. +Note that borgmatic does not run `error` hooks if an error occurs within an +`everything` hook. + There's also another command hook that works a little differently: ```yaml @@ -91,10 +97,10 @@ borgmatic for the `create` and `prune` actions. Here's the order of execution: * Run `before: dump_data_sources` hooks (e.g. for the PostgreSQL hook). * Actually dump data sources (e.g. PostgreSQL databases). * Run `after: dump_data_sources` hooks (e.g. for the PostgreSQL hook). - * Actually run the `create` action. + * Actually run the `create` action (e.g. `borg create`). * Run `after: action` hooks for `create`. * Run `before: action` hooks for `prune`. - * Actually run the `prune` action. + * Actually run the `prune` action (e.g. `borg prune`). * Run `after: action` hooks for `prune`. * Run `after: repository` hooks (for the first repository). * Run `after: configuration` hooks (from the first configuration file). @@ -149,53 +155,13 @@ but not if an error occurs in a previous hook or in the backups themselves. (Prior to borgmatic 1.6.0, these hooks instead ran once per configuration file rather than once per repository.) - -## Variable interpolation - -The command action hooks support interpolating particular runtime variables into -the commands that are run. Here's an example that assumes you provide a separate -shell script: - -```yaml -commands: - - after: action - when: [prune] - run: - - record-prune.sh "{configuration_filename}" "{repository}" -``` - -In this example, when the hook is triggered, borgmatic interpolates runtime -values into the hook command: the borgmatic configuration filename and the -paths of the current Borg repository. Here's the full set of supported -variables you can use here: - - * `configuration_filename`: borgmatic configuration filename in which the - hook was defined - * `log_file` - New in version 1.7.12: - path of the borgmatic log file, only set when the `--log-file` flag is used - * `repository`: path of the current repository as configured in the current - borgmatic configuration file - * `repository_label` New in version - 1.8.12: label of the current repository as configured in the current - borgmatic configuration file - -Not all command hooks support all variables. For instance, the `everything` and -`configuration` hooks don't support repository variables because those hooks -don't run in the context of a single repository. - -Note that you can also interpolate in [arbitrary environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). - - -## Global hooks - You can also use `before_everything` and `after_everything` hooks to perform global setup or cleanup: ```yaml before_everything: - set-up-stuff-globally + after_everything: - clean-up-stuff-globally ``` @@ -213,13 +179,102 @@ but only if there is a `create` action. It runs even if an error occurs during a backup or a backup hook, but not if an error occurs during a `before_everything` hook. +`on_error` hooks run when an error occurs, but only if there is a `create`, +`prune`, `compact`, or `check` action. For instance, borgmatic can run +configurable shell commands to fire off custom error notifications or take other +actions, so you can get alerted as soon as something goes wrong. Here's a +not-so-useful example: -## Error hooks +```yaml +on_error: + - echo "Error while creating a backup or running a backup hook." +``` -borgmatic also runs `on_error` hooks if an error occurs, either when creating -a backup or running a backup hook. See the [monitoring and alerting -documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/) -for more information. +Prior to version 1.8.0 Put +this option in the `hooks:` section of your configuration. + +The `on_error` hook supports interpolating particular runtime variables into +the hook command. Here's an example that assumes you provide a separate shell +script to handle the alerting: + +```yaml +on_error: + - send-text-message.sh +``` + +borgmatic does not run `on_error` hooks if an error occurs within a +`before_everything` or `after_everything` hook. + + +## Variable interpolation + +The command action hooks support interpolating particular runtime variables into +the commands that are run. Here's are a couple examples that assume you provide +separate shell scripts: + +```yaml +commands: + - after: action + when: [prune] + run: + - record-prune.sh {configuration_filename} {repository} + - after: error + when: [create] + run: + - send-text-message.sh {configuration_filename} {repository} +``` + +In this example, when the hook is triggered, borgmatic interpolates runtime +values into each hook command: the borgmatic configuration filename and the +paths of the current Borg repository. + +Here's the full set of supported variables you can use here: + + * `configuration_filename`: borgmatic configuration filename in which the + hook was defined + * `log_file` + New in version 1.7.12: + path of the borgmatic log file, only set when the `--log-file` flag is used + * `repository`: path of the current repository as configured in the current + borgmatic configuration file, if applicable to the current hook + * `repository_label` New in version + 1.8.12: label of the current repository as configured in the current + borgmatic configuration file, if applicable to the current hook + * `error`: the error message itself, only applies to `error` hooks + * `output`: output of the command that failed, only applies to `error` hooks + (may be blank if an error occurred without running a command) + +Not all command hooks support all variables. For instance, the `everything` and +`configuration` hooks don't support repository variables because those hooks +don't run in the context of a single repository. But the deprecated command +hooks (`before_backup`, `on_error`, etc.) do generally support variable +interpolation. + +borgmatic automatically escapes these interpolated values to prevent shell +injection attacks. One implication is that you shouldn't wrap the interpolated +values in your own quotes, as that will interfere with the quoting performed by +borgmatic and result in your command receiving incorrect arguments. For +instance, this won't work: + +```yaml +commands: + - after: error + run: + # Don't do this! It won't work, as the {error} value is already quoted. + - send-text-message.sh "Uh oh: {error}" +``` + +Do this instead: + +```yaml +commands: + - after: error + run: + - send-text-message.sh {error} +``` + +Note that you can also interpolate [arbitrary environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). ## Hook output diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 9bac14d6..9cdbd4be 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -14,140 +14,55 @@ and alerting comes in. There are several different ways you can monitor your backups and find out whether they're succeeding. Which of these you choose to do is up to you and -your particular infrastructure. +your particular infrastructure: -### Job runner alerts - -The easiest place to start is with failure alerts from the [scheduled job -runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) -(cron, systemd, etc.) that's running borgmatic. But note that if the job -doesn't even get scheduled (e.g. due to the job runner not running), you -probably won't get an alert at all! Still, this is a decent first line of -defense, especially when combined with some of the other approaches below. - -### Commands run on error - -The `on_error` hook allows you to run an arbitrary command or script when -borgmatic itself encounters an error running your backups. So for instance, -you can run a script to send yourself a text message alert. But note that if -borgmatic doesn't actually run, this alert won't fire. See [error -hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) -below for how to configure this. - -### Third-party monitoring services - -borgmatic integrates with these monitoring services and libraries, pinging -them as backups happen: - - * [Apprise](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook) - * [Cronhub](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook) - * [Cronitor](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) - * [Grafana Loki](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook) - * [Healthchecks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) - * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook) - * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) - * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook) - * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook) - * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook) - * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook) - -The idea is that you'll receive an alert when something goes wrong or when the -service doesn't hear from borgmatic for a configured interval (if supported). -See the documentation links above for configuration information. - -While these services and libraries offer different features, you probably only -need to use one of them at most. - - -### Third-party monitoring software - -You can use traditional monitoring software to consume borgmatic JSON output -and track when the last successful backup occurred. See [scripting -borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) -below for how to configure this. - -### Borg hosting providers - -Most [Borg hosting -providers](https://torsion.org/borgmatic/#hosting-providers) include -monitoring and alerting as part of their offering. This gives you a dashboard -to check on all of your backups, and can alert you if the service doesn't hear -from borgmatic for a configured interval. - -### Consistency checks - -While not strictly part of monitoring, if you want confidence that your -backups are not only running but are restorable as well, you can configure -particular [consistency -checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) -or even script full [extract -tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). - - -## Error hooks - -When an error occurs during a `create`, `prune`, `compact`, or `check` action, -borgmatic can run configurable shell commands to fire off custom error -notifications or take other actions, so you can get alerted as soon as -something goes wrong. Here's a not-so-useful example: - -```yaml -on_error: - - echo "Error while creating a backup or running a backup hook." -``` - -Prior to version 1.8.0 Put -this option in the `hooks:` section of your configuration. - -The `on_error` hook supports interpolating particular runtime variables into -the hook command. Here's an example that assumes you provide a separate shell -script to handle the alerting: - -```yaml -on_error: - - send-text-message.sh {configuration_filename} {repository} -``` - -In this example, when the error occurs, borgmatic interpolates runtime values -into the hook command: the borgmatic configuration filename and the path of -the repository. Here's the full set of supported variables you can use here: - - * `configuration_filename`: borgmatic configuration filename in which the - error occurred - * `repository`: path of the repository in which the error occurred (may be - blank if the error occurs in a hook) - * `error`: the error message itself - * `output`: output of the command that failed (may be blank if an error - occurred without running a command) - -Note that borgmatic runs the `on_error` hooks only for `create`, `prune`, -`compact`, or `check` actions/hooks in which an error occurs and not other -actions. borgmatic does not run `on_error` hooks if an error occurs within a -`before_everything` or `after_everything` hook. For more about hooks, see the -[borgmatic hooks -documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/), -especially the security information. - -New in version 1.8.7 borgmatic -automatically escapes these interpolated values to prevent shell injection -attacks. One implication of this change is that you shouldn't wrap the -interpolated values in your own quotes, as that will interfere with the -quoting performed by borgmatic and result in your command receiving incorrect -arguments. For instance, this won't work: - - -```yaml -on_error: - # Don't do this! It won't work, as the {error} value is already quoted. - - send-text-message.sh "Uh oh: {error}" -``` - -Do this instead: - -```yaml -on_error: - - send-text-message.sh {error} -``` + * **Job runner alerts**: The easiest place to start is with failure alerts from + the [scheduled job + runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) + (cron, systemd, etc.) that's running borgmatic. But note that if the job + doesn't even get scheduled (e.g. due to the job runner not running), you + probably won't get an alert at all! Still, this is a decent first line of + defense, especially when combined with some of the other approaches below. + * **Third-party monitoring services:** borgmatic integrates with these monitoring + services and libraries, pinging them as backups happen. The idea is that + you'll receive an alert when something goes wrong or when the service doesn't + hear from borgmatic for a configured interval (if supported). While these + services and libraries offer different features, you probably only need to + use one of them at most. See these documentation links for configuration + information: + * [Apprise](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook) + * [Cronhub](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook) + * [Cronitor](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) + * [Grafana Loki](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook) + * [Healthchecks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) + * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook) + * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) + * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook) + * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook) + * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook) + * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook) + * **Third-party monitoring software:** You can use traditional monitoring + software to consume borgmatic JSON output and track when the last successful + backup occurred. See [scripting + borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) + below for how to configure this. + * **Borg hosting providers:** Some [Borg hosting + providers](https://torsion.org/borgmatic/#hosting-providers) include + monitoring and alerting as part of their offering. This gives you a dashboard + to check on all of your backups, and can alert you if the service doesn't + hear from borgmatic for a configured interval. + * **Consistency checks:** While not strictly part of monitoring, if you want + confidence that your backups are not only running but are restorable as well, + you can configure particular [consistency + checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) + or even script full [extract + tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). + * **Commands run on error:** borgmatic's command hooks support running + arbitrary commands or scripts when borgmatic itself encounters an error + running your backups. So for instance, you can run a script to send yourself + a text message alert. But note that if borgmatic doesn't actually run, this + alert won't fire. See the [documentation on command hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/) + for details. ## Healthchecks hook