diff --git a/NEWS b/NEWS
index e77e1003..5a1bd387 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,7 @@
1.9.14.dev0
+ * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the
+ incident UI. See the documentation for more information:
+ https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook
* #936: Clarify Zabbix monitoring hook documentation about creating items.
1.9.13
diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml
index de8dec6f..f6ec574e 100644
--- a/borgmatic/config/schema.yaml
+++ b/borgmatic/config/schema.yaml
@@ -2279,6 +2279,12 @@ properties:
PagerDuty integration key used to notify PagerDuty when a
backup errors. Supports the "{credential ...}" syntax.
example: a177cad45bd374409f78906a810a3074
+ send_logs:
+ type: boolean
+ description: |
+ Send borgmatic logs to PagerDuty when a backup errors.
+ Defaults to true.
+ example: false
description: |
Configuration for a monitoring integration with PagerDuty. Create an
account at https://www.pagerduty.com if you'd like to use this
diff --git a/borgmatic/hooks/monitoring/logs.py b/borgmatic/hooks/monitoring/logs.py
index de1bef60..a2e2f39f 100644
--- a/borgmatic/hooks/monitoring/logs.py
+++ b/borgmatic/hooks/monitoring/logs.py
@@ -64,7 +64,7 @@ def get_handler(identifier):
def format_buffered_logs_for_payload(identifier):
'''
Get the handler previously added to the root logger, and slurp buffered logs out of it to
- send to Healthchecks.
+ send to the monitoring service.
'''
try:
buffering_handler = get_handler(identifier)
diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py
index e238957b..d98375d9 100644
--- a/borgmatic/hooks/monitoring/pagerduty.py
+++ b/borgmatic/hooks/monitoring/pagerduty.py
@@ -6,20 +6,36 @@ import platform
import requests
import borgmatic.hooks.credential.parse
+import borgmatic.hooks.monitoring.logs
from borgmatic.hooks.monitoring import monitor
logger = logging.getLogger(__name__)
EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue'
+DEFAULT_LOGS_PAYLOAD_LIMIT_BYTES = 10000
+HANDLER_IDENTIFIER = 'pagerduty'
-def initialize_monitor(
- integration_key, config, config_filename, monitoring_log_level, dry_run
-): # pragma: no cover
+def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
'''
- No initialization is necessary for this monitor.
+ Add a handler to the root logger that stores in memory the most recent logs emitted. That way,
+ we can send them all to PagerDuty upon a failure state. But skip this if the "send_logs" option
+ is false.
'''
- pass
+ if hook_config.get('send_logs') is False:
+ return
+
+ ping_body_limit = max(
+ DEFAULT_LOGS_PAYLOAD_LIMIT_BYTES
+ - len(borgmatic.hooks.monitoring.logs.PAYLOAD_TRUNCATION_INDICATOR),
+ 0,
+ )
+
+ borgmatic.hooks.monitoring.logs.add_handler(
+ borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler(
+ HANDLER_IDENTIFIER, ping_body_limit, monitoring_log_level
+ )
+ )
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
@@ -37,9 +53,6 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
dry_run_label = ' (dry run; not actually sending)' if dry_run else ''
logger.info(f'Sending failure event to PagerDuty {dry_run_label}')
- if dry_run:
- return
-
try:
integration_key = borgmatic.hooks.credential.parse.resolve_credential(
hook_config.get('integration_key'), config
@@ -48,6 +61,10 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
logger.warning(f'PagerDuty credential error: {error}')
return
+ logs_payload = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload(
+ HANDLER_IDENTIFIER
+ )
+
hostname = platform.node()
local_timestamp = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat()
payload = json.dumps(
@@ -66,11 +83,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
'hostname': hostname,
'configuration filename': config_filename,
'server time': local_timestamp,
+ 'logs': logs_payload,
},
},
}
)
- logger.debug(f'Using PagerDuty payload: {payload}')
+
+ if dry_run:
+ return
logging.getLogger('urllib3').setLevel(logging.ERROR)
try:
@@ -83,6 +103,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover
'''
- No destruction is necessary for this monitor.
+ Remove the monitor handler that was added to the root logger. This prevents the handler from
+ getting reused by other instances of this monitor.
'''
- pass
+ borgmatic.hooks.monitoring.logs.remove_handler(HANDLER_IDENTIFIER)
diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md
index 1f0c959e..9bac14d6 100644
--- a/docs/how-to/monitor-your-backups.md
+++ b/docs/how-to/monitor-your-backups.md
@@ -292,6 +292,27 @@ If you have any issues with the integration, [please contact
us](https://torsion.org/borgmatic/#support-and-contributing).
+### Sending logs
+
+New in version 1.9.14 borgmatic
+logs are included in the payload data sent to PagerDuty. This means that
+(truncated) borgmatic logs, including error messages, show up in the PagerDuty
+incident UI and corresponding notification emails.
+
+You can customize the verbosity of the logs that are sent with borgmatic's
+`--monitoring-verbosity` flag. The `--list` and `--stats` flags may also be of
+use. See `borgmatic create --help` for more information.
+
+If you don't want any logs sent, you can disable this feature by setting
+`send_logs` to `false`:
+
+```yaml
+pagerduty:
+ integration_key: a177cad45bd374409f78906a810a3074
+ send_logs: false
+```
+
+
## Pushover hook
New in version 1.9.2
diff --git a/tests/unit/hooks/monitoring/test_pagerduty.py b/tests/unit/hooks/monitoring/test_pagerduty.py
index f7c73dde..34c55d95 100644
--- a/tests/unit/hooks/monitoring/test_pagerduty.py
+++ b/tests/unit/hooks/monitoring/test_pagerduty.py
@@ -3,6 +3,46 @@ from flexmock import flexmock
from borgmatic.hooks.monitoring import pagerduty as module
+def mock_logger():
+ logger = flexmock()
+ logger.should_receive('addHandler')
+ logger.should_receive('removeHandler')
+ flexmock(module.logging).should_receive('getLogger').and_return(logger)
+
+
+def test_initialize_monitor_creates_log_handler():
+ monitoring_log_level = 1
+
+ mock_logger()
+ flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+ 'Forgetful_buffering_handler'
+ ).once()
+
+ module.initialize_monitor({}, {}, 'test.yaml', monitoring_log_level, dry_run=False)
+
+
+def test_initialize_monitor_creates_log_handler_when_send_logs_true():
+ mock_logger()
+ flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+ 'Forgetful_buffering_handler'
+ ).once()
+
+ module.initialize_monitor(
+ {'send_logs': True}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False
+ )
+
+
+def test_initialize_monitor_bails_when_send_logs_false():
+ mock_logger()
+ flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+ 'Forgetful_buffering_handler'
+ ).never()
+
+ module.initialize_monitor(
+ {'send_logs': False}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False
+ )
+
+
def test_ping_monitor_ignores_start_state():
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
@@ -39,6 +79,9 @@ def test_ping_monitor_calls_api_for_fail_state():
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
+ flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+ 'format_buffered_logs_for_payload'
+ ).and_return('loggy\nlogs')
flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True))
module.ping_monitor(
@@ -55,6 +98,9 @@ def test_ping_monitor_dry_run_does_not_call_api():
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
+ flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+ 'format_buffered_logs_for_payload'
+ ).and_return('loggy\nlogs')
flexmock(module.requests).should_receive('post').never()
module.ping_monitor(
@@ -71,6 +117,9 @@ def test_ping_monitor_with_connection_error_logs_warning():
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
+ flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+ 'format_buffered_logs_for_payload'
+ ).and_return('loggy\nlogs')
flexmock(module.requests).should_receive('post').and_raise(
module.requests.exceptions.ConnectionError
)
@@ -108,6 +157,9 @@ def test_ping_monitor_with_other_error_logs_warning():
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
+ flexmock(module.borgmatic.hooks.monitoring.logs).should_receive(
+ 'format_buffered_logs_for_payload'
+ ).and_return('loggy\nlogs')
response.should_receive('raise_for_status').and_raise(
module.requests.exceptions.RequestException
)