Code formatting.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/976

NEWS

@@ -1,3 +1,12 @@
+1.9.7
+ * #855: Add a Sentry monitoring hook. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook
+ * #968: Fix for a "spot" check error when a filename in the most recent archive contains a newline.
+ * #970: Fix for an error when there's a blank line in the configured patterns or excludes.
+ * #971: Fix for "exclude_from" files being completely ignored.
+ * #977: Fix for "exclude_patterns" and "exclude_from" not supporting explicit pattern styles (e.g.,
+   "sh:" or "re:").
+
 1.9.6
  * #959: Fix an error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's
    source directories.

README.md

@@ -75,6 +75,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
+<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
 <a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
 
 

borgmatic/actions/check.py

@@ -401,7 +401,7 @@ def collect_spot_check_source_paths(
 
     paths = tuple(
         path_line.split(' ', 1)[1]
-        for path_line in paths_output.split('\n')
+        for path_line in paths_output.splitlines()
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
 

borgmatic/actions/create.py

@@ -15,7 +15,7 @@ import borgmatic.hooks.dispatch
 logger = logging.getLogger(__name__)
 
 
-def parse_pattern(pattern_line):
+def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_style.NONE):
     '''
     Given a Borg pattern as a string, parse it into a borgmatic.borg.pattern.Pattern instance and
     return it.
@@ -23,12 +23,13 @@ def parse_pattern(pattern_line):
     try:
         (pattern_type, remainder) = pattern_line.split(' ', maxsplit=1)
     except ValueError:
-        raise ValueError('Invalid pattern:', pattern_line)
+        raise ValueError(f'Invalid pattern: {pattern_line}')
 
     try:
-        (pattern_style, path) = remainder.split(':', maxsplit=1)
+        (parsed_pattern_style, path) = remainder.split(':', maxsplit=1)
+        pattern_style = borgmatic.borg.pattern.Pattern_style(parsed_pattern_style)
     except ValueError:
-        pattern_style = ''
+        pattern_style = default_style
         path = remainder
 
     return borgmatic.borg.pattern.Pattern(
@@ -54,39 +55,40 @@ def collect_patterns(config):
                 for source_directory in config.get('source_directories', ())
             )
             + tuple(
-                parse_pattern(pattern_line.strip())
+                parse_pattern(pattern_line.strip(), borgmatic.borg.pattern.Pattern_style.SHELL)
                 for pattern_line in config.get('patterns', ())
                 if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
             )
             + tuple(
-                borgmatic.borg.pattern.Pattern(
-                    exclude_line.strip(),
-                    borgmatic.borg.pattern.Pattern_type.EXCLUDE,
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
                     borgmatic.borg.pattern.Pattern_style.FNMATCH,
                 )
                 for exclude_line in config.get('exclude_patterns', ())
             )
             + tuple(
-                parse_pattern(pattern_line.strip())
+                parse_pattern(pattern_line.strip(), borgmatic.borg.pattern.Pattern_style.SHELL)
                 for filename in config.get('patterns_from', ())
                 for pattern_line in open(filename).readlines()
                 if not pattern_line.lstrip().startswith('#')
+                if pattern_line.strip()
             )
             + tuple(
-                borgmatic.borg.pattern.Pattern(
-                    exclude_line.strip(),
-                    borgmatic.borg.pattern.Pattern_type.EXCLUDE,
+                parse_pattern(
+                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
                     borgmatic.borg.pattern.Pattern_style.FNMATCH,
                 )
-                for filename in config.get('excludes_from', ())
+                for filename in config.get('exclude_from', ())
                 for exclude_line in open(filename).readlines()
                 if not exclude_line.lstrip().startswith('#')
+                if exclude_line.strip()
             )
         )
     except (FileNotFoundError, OSError) as error:
         logger.debug(error)
 
-        raise ValueError(f'Cannot read patterns_from/excludes_from file: {error.filename}')
+        raise ValueError(f'Cannot read patterns_from/exclude_from file: {error.filename}')
 
 
 def expand_directory(directory, working_directory):

borgmatic/borg/list.py

@@ -120,7 +120,7 @@ def capture_archive_listing(
                     paths=[path for path in list_paths] if list_paths else None,
                     find_paths=None,
                     json=None,
-                    format=path_format or '{path}{NL}',  # noqa: FS003
+                    format=path_format or '{path}{NUL}',  # noqa: FS003
                 ),
                 global_arguments,
                 local_path,
@@ -132,7 +132,7 @@ def capture_archive_listing(
             borg_exit_codes=config.get('borg_exit_codes'),
         )
         .strip('\n')
-        .split('\n')
+        .split('\0')
     )
 
 
@@ -230,7 +230,7 @@ def list_archive(
                 borg_exit_codes=borg_exit_codes,
             )
             .strip('\n')
-            .split('\n')
+            .splitlines()
         )
     else:
         archive_lines = (list_arguments.archive,)

borgmatic/config/schema.yaml

@@ -2259,7 +2259,45 @@ properties:
             can send the logs to a self-hosted instance or create an account at
             https://grafana.com/auth/sign-up/create-user. See borgmatic
             monitoring documentation for details.
-
+    sentry:
+        type: object
+        required: ['data_source_name_url', 'monitor_slug']
+        additionalProperties: false
+        properties:
+            data_source_name_url:
+                type: string
+                description: |
+                    Sentry Data Source Name (DSN) URL, associated with a
+                    particular Sentry project. Used to construct a cron URL,
+                    notified when a backup begins, ends, or errors.
+                example: https://5f80ec@o294220.ingest.us.sentry.io/203069
+            monitor_slug:
+                type: string
+                description: |
+                    Sentry monitor slug, associated with a particular Sentry
+                    project monitor. Used along with the data source name URL to
+                    construct a cron URL.
+                example: mymonitor
+            states:
+                type: array
+                items:
+                    type: string
+                    enum:
+                        - start
+                        - finish
+                        - fail
+                    uniqueItems: true
+                description: |
+                    List of one or more monitoring states to ping for: "start",
+                    "finish", and/or "fail". Defaults to pinging for all states.
+                example:
+                    - start
+                    - finish
+        description: |
+            Configuration for a monitoring integration with Sentry. You can use
+            a self-hosted instance via https://develop.sentry.dev/self-hosted/
+            or create a cloud-hosted account at https://sentry.io. See borgmatic
+            monitoring documentation for details.
     zfs:
         type: ["object", "null"]
         additionalProperties: false

borgmatic/hooks/monitoring/ntfy.py

@@ -19,7 +19,6 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
     Ping the configured Ntfy topic. Use the given configuration filename in any log entries.
     If this is a dry run, then don't actually ping anything.
     '''
-
     run_states = hook_config.get('states', ['fail'])
 
     if state.name.lower() in run_states:

borgmatic/hooks/monitoring/sentry.py

@@ -0,0 +1,79 @@
+import logging
+import re
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def initialize_monitor(
+    ping_url, config, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
+DATA_SOURCE_NAME_URL_PATTERN = re.compile(
+    '^(?P<protocol>.*)://(?P<username>.*)@(?P<hostname>.*)/(?P<project_id>.*)$'
+)
+
+
+def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    Construct and ping a Sentry cron URL, based on the configured DSN URL and monitor slug. Use the
+    given configuration filename in any log entries. If this is a dry run, then don't actually ping
+    anything.
+    '''
+    run_states = hook_config.get('states', ['start', 'finish', 'fail'])
+
+    if not state.name.lower() in run_states:
+        return
+
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+    data_source_name_url = hook_config.get('data_source_name_url')
+    monitor_slug = hook_config.get('monitor_slug')
+    match = DATA_SOURCE_NAME_URL_PATTERN.match(data_source_name_url)
+
+    if not match:
+        logger.warning(
+            'f{config_filename}: Invalid Sentry data source name URL: {data_source_name_url}'
+        )
+        return
+
+    cron_url = f'{match.group("protocol")}://{match.group("hostname")}/api/{match.group("project_id")}/cron/{monitor_slug}/{match.group("username")}/'
+
+    logger.info(f'{config_filename}: Pinging Sentry {state.name.lower()}{dry_run_label}')
+    logger.debug(f'{config_filename}: Using Sentry cron URL {cron_url}')
+
+    status = {
+        'start': 'in_progress',
+        'finish': 'ok',
+        'fail': 'error',
+    }.get(state.name.lower())
+
+    if not status:
+        logger.warning('f{config_filename}: Invalid Sentry state')
+        return
+
+    if dry_run:
+        return
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    try:
+        response = requests.post(f'{cron_url}?status={status}')
+        if not response.ok:
+            response.raise_for_status()
+    except requests.exceptions.RequestException as error:
+        logger.warning(f'{config_filename}: Sentry error: {error}')
+
+
+def destroy_monitor(
+    ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No destruction is necessary for this monitor.
+    '''
+    pass

docs/how-to/monitor-your-backups.md

@@ -47,6 +47,7 @@ them as backups happen:
  * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook)
  * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook)
  * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook)
+ * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook)
  * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook)
  * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook)
 
@@ -361,10 +362,49 @@ pushover:
 ```
 
 
+## Sentry hook
+
+<span class="minilink minilink-addedin">New in version 1.9.7</span>
+[Sentry](https://sentry.io/) is an application monitoring service that
+includes cron-style monitoring (either cloud-hosted or
+[self-hosted](https://develop.sentry.dev/self-hosted/)).
+
+To get started, create a [Sentry cron
+monitor](https://docs.sentry.io/product/crons/) in the Sentry UI. Under
+"Instrument your monitor," select "Sentry CLI" and copy the URL value for the
+displayed
+[`SENTRY_DSN`](https://docs.sentry.io/concepts/key-terms/dsn-explainer/)
+environment variable into borgmatic's Sentry `data_source_name_url`
+configuration option. For example:
+
+```
+sentry:
+    data_source_name_url: https://5f80ec@o294220.ingest.us.sentry.io/203069
+    monitor_slug: mymonitor
+```
+
+The `monitor_slug` value comes from the "Monitor Slug" under "Cron Details" on
+the same Sentry monitor page.
+
+With this configuration, borgmatic pings Sentry whenever borgmatic starts,
+finishes, or fails, but only when any of the `create`, `prune`, `compact`, or
+`check` actions are run. You can optionally override the start/finish/fail
+behavior with the `states` configuration option. For instance, to only ping
+Sentry on failure:
+
+```
+sentry:
+    data_source_name_url: https://5f80ec@o294220.ingest.us.sentry.io/203069
+    monitor_slug: mymonitor
+    states:
+      - fail
+```
+
+
 ## ntfy hook
 
 <span class="minilink minilink-addedin">New in version 1.6.3</span>
-[ntfy](https://ntfy.sh) is a free, simple, service (either hosted or
+[ntfy](https://ntfy.sh) is a free, simple, service (either cloud-hosted or
 self-hosted) which offers simple pub/sub push notifications to multiple
 platforms including [web](https://ntfy.sh/stats),
 [Android](https://play.google.com/store/apps/details?id=io.heckel.ntfy) and

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "1.9.6"
+version = "1.9.7"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]

tests/unit/actions/test_create.py

@@ -34,20 +34,27 @@ def test_collect_patterns_converts_source_directories():
 
 
 def test_collect_patterns_parses_config_patterns():
-    flexmock(module).should_receive('parse_pattern').with_args('R /foo').and_return(Pattern('/foo'))
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /foo', Pattern_style.SHELL
+    ).and_return(Pattern('/foo'))
     flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
-    flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /bar', Pattern_style.SHELL
+    ).and_return(Pattern('/bar'))
 
-    assert module.collect_patterns({'patterns': ['R /foo', '# comment', 'R /bar']}) == (
+    assert module.collect_patterns({'patterns': ['R /foo', '# comment', '', '   ', 'R /bar']}) == (
         Pattern('/foo'),
         Pattern('/bar'),
     )
 
 
 def test_collect_patterns_converts_exclude_patterns():
-    assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar']}) == (
+    assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
         Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
         Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
+        Pattern('**/baz', Pattern_type.EXCLUDE, Pattern_style.SHELL),
     )
 
 
@@ -55,12 +62,20 @@ def test_collect_patterns_reads_config_patterns_from_file():
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('R /foo'))
     builtins.should_receive('open').with_args('file2.txt').and_return(
-        io.StringIO('R /bar\n# comment\nR /baz')
+        io.StringIO('R /bar\n# comment\n\n   \nR /baz')
     )
-    flexmock(module).should_receive('parse_pattern').with_args('R /foo').and_return(Pattern('/foo'))
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /foo', Pattern_style.SHELL
+    ).and_return(Pattern('/foo'))
     flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
-    flexmock(module).should_receive('parse_pattern').with_args('R /bar').and_return(Pattern('/bar'))
-    flexmock(module).should_receive('parse_pattern').with_args('R /baz').and_return(Pattern('/baz'))
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /bar', Pattern_style.SHELL
+    ).and_return(Pattern('/bar'))
+    flexmock(module).should_receive('parse_pattern').with_args(
+        'R /baz', Pattern_style.SHELL
+    ).and_return(Pattern('/baz'))
 
     assert module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']}) == (
         Pattern('/foo'),
@@ -78,27 +93,39 @@ def test_collect_patterns_errors_on_missing_config_patterns_from_file():
         module.collect_patterns({'patterns_from': ['file1.txt', 'file2.txt']})
 
 
-def test_collect_patterns_reads_config_excludes_from_file():
+def test_collect_patterns_reads_config_exclude_from_file():
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('file1.txt').and_return(io.StringIO('/foo'))
     builtins.should_receive('open').with_args('file2.txt').and_return(
-        io.StringIO('/bar\n# comment\n/baz')
+        io.StringIO('/bar\n# comment\n\n   \n/baz')
     )
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '- /foo', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '- /bar', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+    flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
+    flexmock(module).should_receive('parse_pattern').with_args('').never()
+    flexmock(module).should_receive('parse_pattern').with_args('   ').never()
+    flexmock(module).should_receive('parse_pattern').with_args(
+        '- /baz', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/baz', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
 
-    assert module.collect_patterns({'excludes_from': ['file1.txt', 'file2.txt']}) == (
+    assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) == (
         Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
         Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
         Pattern('/baz', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
     )
 
 
-def test_collect_patterns_errors_on_missing_config_excludes_from_file():
+def test_collect_patterns_errors_on_missing_config_exclude_from_file():
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('file1.txt').and_raise(OSError)
     flexmock(module).should_receive('parse_pattern').never()
 
     with pytest.raises(ValueError):
-        module.collect_patterns({'excludes_from': ['file1.txt', 'file2.txt']})
+        module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']})
 
 
 def test_expand_directory_with_basic_path_passes_it_through():

tests/unit/hooks/monitoring/test_sentry.py

@@ -0,0 +1,124 @@
+import pytest
+from flexmock import flexmock
+
+import borgmatic.hooks.monitoring.monitor
+from borgmatic.hooks.monitoring import sentry as module
+
+
+@pytest.mark.parametrize(
+    'state,configured_states,expected_status',
+    (
+        (borgmatic.hooks.monitoring.monitor.State.START, ['start'], 'in_progress'),
+        (
+            borgmatic.hooks.monitoring.monitor.State.START,
+            ['start', 'finish', 'fail'],
+            'in_progress',
+        ),
+        (borgmatic.hooks.monitoring.monitor.State.START, None, 'in_progress'),
+        (borgmatic.hooks.monitoring.monitor.State.FINISH, ['finish'], 'ok'),
+        (borgmatic.hooks.monitoring.monitor.State.FAIL, ['fail'], 'error'),
+    ),
+)
+def test_ping_monitor_constructs_cron_url_and_pings_it(state, configured_states, expected_status):
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+    }
+
+    if configured_states:
+        hook_config['states'] = configured_states
+
+    flexmock(module.requests).should_receive('get').with_args(
+        f'https://o294220.ingest.us.sentry.io/api/203069/cron/test/5f80ec/?status={expected_status}'
+    ).and_return(flexmock(ok=True))
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        state,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_unconfigured_state_bails():
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+        'states': ['fail'],
+    }
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+@pytest.mark.parametrize(
+    'data_source_name_url',
+    (
+        '5f80ec@o294220.ingest.us.sentry.io/203069',
+        'https://o294220.ingest.us.sentry.io/203069',
+        'https://5f80ec@/203069',
+        'https://5f80ec@o294220.ingest.us.sentry.io',
+        'https://5f80ec@o294220.ingest.us.sentry.io/203069/',
+    ),
+)
+def test_ping_monitor_with_invalid_data_source_name_url_bails(data_source_name_url):
+    hook_config = {
+        'data_source_name_url': data_source_name_url,
+        'monitor_slug': 'test',
+    }
+
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_invalid_sentry_state_bails():
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+        # This should never actually happen in practice, because the config schema prevents it.
+        'states': ['log'],
+    }
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.LOG,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_dry_run_bails():
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+    }
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=True,
+    )