Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/543

NEWS

@@ -1,3 +1,15 @@
+1.6.3
+ * #541: Add "borgmatic list --find" flag for searching for files across multiple archives, useful
+   for hunting down that file you accidentally deleted so you can extract it. See the documentation
+   for more information:
+   https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#searching-for-a-file
+ * #543: Add a monitoring hook for sending push notifications via ntfy. See the documentation for
+   more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook
+ * Fix Bash completion script to no longer alter your shell's settings (complain about unset
+   variables or error on pipe failures).
+ * Deprecate "borgmatic list --successful" flag, as listing only non-checkpoint (successful)
+   archives is now the default in newer versions of Borg.
+
 1.6.2
  * #523: Reduce the default consistency check frequency and support configuring the frequency
    independently for each check. Also add "borgmatic check --force" flag to ignore configured

README.md

@@ -71,6 +71,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
 <a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 <a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px;"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
 
 

borgmatic/borg/list.py

@@ -1,4 +1,6 @@
+import copy
 import logging
+import re
 
 from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
@@ -6,17 +8,11 @@ from borgmatic.execute import execute_command
 logger = logging.getLogger(__name__)
 
 
-# A hack to convince Borg to exclude archives ending in ".checkpoint". This assumes that a
-# non-checkpoint archive name ends in a digit (e.g. from a timestamp).
-BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
-
-
 def resolve_archive_name(repository, archive, storage_config, local_path='borg', remote_path=None):
     '''
     Given a local or remote repository path, an archive name, a storage config dict, a local Borg
     path, and a remote Borg path, simply return the archive name. But if the archive name is
-    "latest", then instead introspect the repository for the latest successful (non-checkpoint)
+    "latest", then instead introspect the repository for the latest archive and return its name.
-    archive, and return its name.
 
     Raise ValueError if "latest" is given but there are no archives in the repository.
     '''
@@ -31,7 +27,6 @@ def resolve_archive_name(repository, archive, storage_config, local_path='borg',
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + make_flags('remote-path', remote_path)
         + make_flags('lock-wait', lock_wait)
-        + make_flags('glob-archives', BORG_EXCLUDE_CHECKPOINTS_GLOB)
         + make_flags('last', 1)
         + ('--short', repository)
     )
@@ -47,17 +42,20 @@ def resolve_archive_name(repository, archive, storage_config, local_path='borg',
     return latest_archive
 
 
-def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
+MAKE_FLAGS_EXCLUDES = ('repository', 'archive', 'successful', 'paths', 'find_paths')
+
+
+def make_list_command(
+    repository, storage_config, list_arguments, local_path='borg', remote_path=None
+):
     '''
-    Given a local or remote repository path, a storage config dict, and the arguments to the list
+    Given a local or remote repository path, a storage config dict, the arguments to the list
-    action, display the output of listing Borg archives in the repository or return JSON output. Or,
+    action, and local and remote Borg paths, return a command as a tuple to list archives or paths
-    if an archive name is given, listing the files in that archive.
+    within an archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
-    if list_arguments.successful:
-        list_arguments.glob_archives = BORG_EXCLUDE_CHECKPOINTS_GLOB
 
-    full_command = (
+    return (
         (local_path, 'list')
         + (
             ('--info',)
@@ -71,19 +69,92 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
         )
         + make_flags('remote-path', remote_path)
         + make_flags('lock-wait', lock_wait)
-        + make_flags_from_arguments(
+        + make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES,)
-            list_arguments, excludes=('repository', 'archive', 'paths', 'successful')
-        )
         + (
-            '::'.join((repository, list_arguments.archive))
+            ('::'.join((repository, list_arguments.archive)),)
             if list_arguments.archive
-            else repository,
+            else (repository,)
         )
         + (tuple(list_arguments.paths) if list_arguments.paths else ())
     )
 
-    return execute_command(
+
-        full_command,
+def make_find_paths(find_paths):
-        output_log_level=None if list_arguments.json else logging.WARNING,
+    '''
-        borg_local_path=local_path,
+    Given a sequence of path fragments or patterns as passed to `--find`, transform all path
+    fragments into glob patterns. Pass through existing patterns untouched.
+
+    For example, given find_paths of:
+
+      ['foo.txt', 'pp:root/somedir']
+
+    ... transform that into:
+
+      ['sh:**/*foo.txt*/**', 'pp:root/somedir']
+    '''
+    if not find_paths:
+        return ()
+
+    return tuple(
+        find_path
+        if re.compile(r'([-!+RrPp] )|(\w\w:)').match(find_path)
+        else f'sh:**/*{find_path}*/**'
+        for find_path in find_paths
     )
+
+
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
+    '''
+    Given a local or remote repository path, a storage config dict, the arguments to the list
+    action, and local and remote Borg paths, display the output of listing Borg archives in the
+    repository or return JSON output. Or, if an archive name is given, list the files in that
+    archive. Or, if list_arguments.find_paths are given, list the files by searching across multiple
+    archives.
+    '''
+    # If there are any paths to find (and there's not a single archive already selected), start by
+    # getting a list of archives to search.
+    if list_arguments.find_paths and not list_arguments.archive:
+        repository_arguments = copy.copy(list_arguments)
+        repository_arguments.archive = None
+        repository_arguments.json = False
+        repository_arguments.format = None
+
+        # Ask Borg to list archives. Capture its output for use below.
+        archive_lines = tuple(
+            execute_command(
+                make_list_command(
+                    repository, storage_config, repository_arguments, local_path, remote_path
+                ),
+                output_log_level=None,
+                borg_local_path=local_path,
+            )
+            .strip('\n')
+            .split('\n')
+        )
+    else:
+        archive_lines = (list_arguments.archive,)
+
+    # For each archive listed by Borg, run list on the contents of that archive.
+    for archive_line in archive_lines:
+        try:
+            archive = archive_line.split()[0]
+        except (AttributeError, IndexError):
+            archive = None
+
+        if archive:
+            logger.warning(archive_line)
+
+        archive_arguments = copy.copy(list_arguments)
+        archive_arguments.archive = archive
+        main_command = make_list_command(
+            repository, storage_config, archive_arguments, local_path, remote_path
+        ) + make_find_paths(list_arguments.find_paths)
+
+        output = execute_command(
+            main_command,
+            output_log_level=None if list_arguments.json else logging.WARNING,
+            borg_local_path=local_path,
+        )
+
+        if list_arguments.json:
+            return output

borgmatic/commands/arguments.py

@@ -554,7 +554,14 @@ def make_parsers():
         metavar='PATH',
         nargs='+',
         dest='paths',
-        help='Paths to list from archive, defaults to the entire archive',
+        help='Paths or patterns to list from a single selected archive (via "--archive"), defaults to listing the entire archive',
+    )
+    list_group.add_argument(
+        '--find',
+        metavar='PATH',
+        nargs='+',
+        dest='find_paths',
+        help='Partial paths or patterns to search for and list across multiple archives',
     )
     list_group.add_argument(
         '--short', default=False, action='store_true', help='Output only archive or path names'
@@ -571,9 +578,9 @@ def make_parsers():
     )
     list_group.add_argument(
         '--successful',
-        default=False,
+        default=True,
         action='store_true',
-        help='Only list archive names of successful (non-checkpoint) backups',
+        help='Deprecated in favor of listing successful (non-checkpoint) backups by default in newer versions of Borg',
     )
     list_group.add_argument(
         '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
@@ -681,9 +688,6 @@ def parse_arguments(*unparsed_arguments):
     if 'init' in arguments and arguments['global'].dry_run:
         raise ValueError('The init action cannot be used with the --dry-run option')
 
-    if 'list' in arguments and arguments['list'].glob_archives and arguments['list'].successful:
-        raise ValueError('The --glob-archives and --successful options cannot be used together')
-
     if (
         'list' in arguments
         and 'info' in arguments

borgmatic/commands/completion.py

@@ -30,7 +30,6 @@ def bash_completion():
     # Avert your eyes.
     return '\n'.join(
         (
-            'set -uo pipefail',
             'check_version() {',
             '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
             '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',

borgmatic/config/schema.yaml

@@ -900,6 +900,107 @@ properties:
                     https://docs.mongodb.com/database-tools/mongodump/ and
                     https://docs.mongodb.com/database-tools/mongorestore/ for
                     details.
+            ntfy:
+                type: object
+                required: ['topic']
+                additionalProperties: false
+                properties:
+                    topic:
+                        type: string
+                        description: |
+                            The topic to publish to
+                            (https://ntfy.sh/docs/publish/)
+                        example: topic
+                    server:
+                        type: string
+                        description: |
+                            The address of your self-hosted ntfy.sh installation
+                        example: https://ntfy.your-domain.com
+                    start:
+                        type: object
+                        properties:
+                            title:
+                                type: string
+                                description: |
+                                    The title of the message
+                                example: Ping!
+                            message:
+                                type: string
+                                description: |
+                                    The message body to publish
+                                example: Your backups have failed.
+                            priority:
+                                type: string
+                                description: |
+                                    The priority to set
+                                example: urgent
+                            tags:
+                                type: string
+                                description: |
+                                    Tags to attach to the message
+                                example: incoming_envelope
+                    finish:
+                        type: object
+                        properties:
+                            title:
+                                type: string
+                                description: |
+                                    The title of the message
+                                example: Ping!
+                            message:
+                                type: string
+                                description: |
+                                    The message body to publish
+                                example: Your backups have failed.
+                            priority:
+                                type: string
+                                description: |
+                                    The priority to set
+                                example: urgent
+                            tags:
+                                type: string
+                                description: |
+                                    Tags to attach to the message
+                                example: incoming_envelope
+                    fail:
+                        type: object
+                        properties:
+                            title:
+                                type: string
+                                description: |
+                                    The title of the message
+                                example: Ping!
+                            message:
+                                type: string
+                                description: |
+                                    The message body to publish
+                                example: Your backups have failed.
+                            priority:
+                                type: string
+                                description: |
+                                    The priority to set
+                                example: urgent
+                            tags:
+                                type: string
+                                description: |
+                                    Tags to attach to the message
+                                example: incoming_envelope
+                    states:
+                        type: array
+                        items:
+                            type: string
+                            enum:
+                                - start
+                                - finish
+                                - fail
+                            uniqueItems: true
+                        description: |
+                            List of one or more monitoring states to ping for:
+                            "start", "finish", and/or "fail". Defaults to
+                            pinging for failure only.
+                        example:
+                            - start
+                            - finish
             healthchecks:
                 type: object
                 required: ['ping_url']

borgmatic/hooks/dispatch.py

@@ -1,17 +1,27 @@
 import logging
 
-from borgmatic.hooks import cronhub, cronitor, healthchecks, mongodb, mysql, pagerduty, postgresql
+from borgmatic.hooks import (
+    cronhub,
+    cronitor,
+    healthchecks,
+    mongodb,
+    mysql,
+    ntfy,
+    pagerduty,
+    postgresql,
+)
 
 logger = logging.getLogger(__name__)
 
 HOOK_NAME_TO_MODULE = {
-    'healthchecks': healthchecks,
-    'cronitor': cronitor,
     'cronhub': cronhub,
+    'cronitor': cronitor,
+    'healthchecks': healthchecks,
+    'mongodb_databases': mongodb,
+    'mysql_databases': mysql,
+    'ntfy': ntfy,
     'pagerduty': pagerduty,
     'postgresql_databases': postgresql,
-    'mysql_databases': mysql,
-    'mongodb_databases': mongodb,
 }
 
 

borgmatic/hooks/monitor.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
-MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty')
+MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy')
 
 
 class State(Enum):

borgmatic/hooks/ntfy.py

@@ -0,0 +1,73 @@
+import logging
+
+import requests
+
+from borgmatic.hooks import monitor
+
+logger = logging.getLogger(__name__)
+
+MONITOR_STATE_TO_NTFY = {
+    monitor.State.START: None,
+    monitor.State.FINISH: None,
+    monitor.State.FAIL: None,
+}
+
+
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
+def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
+    '''
+    Ping the configured Ntfy topic. Use the given configuration filename in any log entries.
+    If this is a dry run, then don't actually ping anything.
+    '''
+
+    run_states = hook_config.get('states', ['fail'])
+
+    if state.name.lower() in run_states:
+        dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+
+        state_config = hook_config.get(
+            state.name.lower(),
+            {
+                'title': f'A Borgmatic {state.name} event happened',
+                'message': f'A Borgmatic {state.name} event happened',
+                'priority': 'default',
+                'tags': 'borgmatic',
+            },
+        )
+
+        base_url = hook_config.get('server', 'https://ntfy.sh')
+        topic = hook_config.get('topic')
+
+        logger.info(f'{config_filename}: Pinging ntfy topic {topic}{dry_run_label}')
+        logger.debug(f'{config_filename}: Using Ntfy ping URL {base_url}/{topic}')
+
+        headers = {
+            'X-Title': state_config.get('title'),
+            'X-Message': state_config.get('message'),
+            'X-Priority': state_config.get('priority'),
+            'X-Tags': state_config.get('tags'),
+        }
+
+        if not dry_run:
+            logging.getLogger('urllib3').setLevel(logging.ERROR)
+            try:
+                requests.post(f'{base_url}/{topic}', headers=headers)
+            except requests.exceptions.RequestException as error:
+                logger.warning(f'{config_filename}: Ntfy error: {error}')
+
+
+def destroy_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No destruction is necessary for this monitor.
+    '''
+    pass

docs/how-to/extract-a-backup.md

@@ -116,7 +116,7 @@ Omit the `--archive` flag to mount all archives (lazy-loaded):
 borgmatic mount --mount-point /mnt
 ```
 
-Or use the "latest" value for the archive to mount the latest successful archive:
+Or use the "latest" value for the archive to mount the latest archive:
 
 ```bash
 borgmatic mount --archive latest --mount-point /mnt

docs/how-to/inspect-your-backups.md

@@ -51,6 +51,31 @@ borgmatic info
 `--info`. Or upgrade borgmatic!)
 
 
+### Searching for a file
+
+Let's say you've accidentally deleted a file and want to find the backup
+archive(s) containing it. `borgmatic list` provides a `--find` flag for
+exactly this purpose (as of borgmatic 1.6.3). For instance, if you're looking
+for a `foo.txt`:
+
+```bash
+borgmatic list --find foo.txt
+```
+
+This will list your archives and indicate those with files matching
+`*foo.txt*` anywhere in the archive. The `--find` parameter can alternatively
+be a [Borg
+pattern](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-patterns).
+
+To limit the archives searched, use the standard `list` parameters for
+filtering archives such as `--last`, `--archive`, `--glob-archives`, etc. For
+example, to search only the last five archives:
+
+```bash
+borgmatic list --find foo.txt --last 5
+```
+
+
 ## Logging
 
 By default, borgmatic logs to a local syslog-compatible daemon if one is

docs/how-to/monitor-your-backups.md

@@ -38,17 +38,19 @@ below for how to configure this.
 
 borgmatic integrates with monitoring services like
 [Healthchecks](https://healthchecks.io/), [Cronitor](https://cronitor.io),
-[Cronhub](https://cronhub.io), and [PagerDuty](https://www.pagerduty.com/) and
+[Cronhub](https://cronhub.io), [PagerDuty](https://www.pagerduty.com/), and
-pings these services whenever borgmatic runs. That way, you'll receive an
+[ntfy](https://ntfy.sh/) and pings these services whenever borgmatic runs.
-alert when something goes wrong or (for certain hooks) the service doesn't
+That way, you'll receive an alert when something goes wrong or (for certain
-hear from borgmatic for a configured interval. See [Healthchecks
+hooks) the service doesn't hear from borgmatic for a configured interval. See
+[Healthchecks
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook),
 [Cronitor
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook),
 [Cronhub
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook),
-and [PagerDuty
+[PagerDuty
-hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook)
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook),
+and [ntfy hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook)
 below for how to configure this.
 
 While these services offer different features, you probably only need to use
@@ -270,6 +272,52 @@ If you have any issues with the integration, [please contact
 us](https://torsion.org/borgmatic/#support-and-contributing).
 
 
+## ntfy hook
+
+[ntfy](https://ntfy.sh) is a free, simple, service (either hosted or self-hosted)
+which offers simple pub/sub push notifications to multiple platforms including
+[web](https://ntfy.sh/stats), [Android](https://play.google.com/store/apps/details?id=io.heckel.ntfy)
+and [iOS](https://apps.apple.com/us/app/ntfy/id1625396347).
+
+Since push notifications for regular events might soon become quite annoying,
+this hook only fires on any errors by default in order to instantly alert you to issues.
+The `states` list can override this.
+
+As ntfy is unauthenticated, it isn't a suitable channel for any private information
+so the default messages are intentionally generic. These can be overridden, depending
+on your risk assessment. Each `state` can have its own custom messages, priorities and tags
+or, if none are provided, will use the default.
+
+An example configuration is shown here, with all the available options, including
+[priorities](https://ntfy.sh/docs/publish/#message-priority) and
+[tags](https://ntfy.sh/docs/publish/#tags-emojis):
+
+```yaml
+hooks:
+    ntfy:
+        topic: my-unique-topic
+        server: https://ntfy.my-domain.com
+        start:
+            title: A Borgmatic backup started
+            message: Watch this space...
+            tags: borgmatic
+            priority: min
+        finish:
+            title: A Borgmatic backup completed successfully
+            message: Nice!
+            tags: borgmatic,+1
+            priority: min
+        fail:
+            title: A Borgmatic backup failed
+            message: You should probably fix it
+            tags: borgmatic,-1,skull
+            priority: max
+        states:
+            - start
+            - finish
+            - fail
+```
+
 ## Scripting borgmatic
 
 To consume the output of borgmatic in other software, you can include an
@@ -286,35 +334,12 @@ output only shows up at the console, and not in syslog.
  * [Borgmacator GNOME AppIndicator](https://github.com/N-Coder/borgmacator/)
 
 
-### Successful backups
-
-`borgmatic list` includes support for a `--successful` flag that only lists
-successful (non-checkpoint) backups. This flag works via a basic heuristic: It
-assumes that non-checkpoint archive names end with a digit (e.g. from a
-timestamp), while checkpoint archive names do not. This means that if you're
-using custom archive names that do not end in a digit, the `--successful` flag
-will not work as expected.
-
-Combined with a built-in Borg flag like `--last`, you can list the last
-successful backup for use in your monitoring scripts. Here's an example
-combined with `--json`:
-
-```bash
-borgmatic list --successful --last 1 --json
-```
-
-Note that this particular combination will only work if you've got a single
-backup "series" in your repository. If you're instead backing up, say, from
-multiple different hosts into a single repository, then you'll need to get
-fancier with your archive listing. See `borg list --help` for more flags.
-
-
 ### Latest backups
 
 All borgmatic actions that accept an "--archive" flag allow you to specify an
-archive name of "latest". This lets you get the latest successful archive
+archive name of "latest". This lets you get the latest archive without having
-without having to first run "borgmatic list" manually, which can be handy in
+to first run "borgmatic list" manually, which can be handy in automated
-automated scripts. Here's an example:
+scripts. Here's an example:
 
 ```bash
 borgmatic info --archive latest

docs/how-to/set-up-backups.md

@@ -92,6 +92,7 @@ installing borgmatic:
  * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
+ * [macOS (via Homebrew)](https://formulae.brew.sh/formula/borgmatic)
  * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup)
  * [virtualenv](https://virtualenv.pypa.io/en/stable/)
 
@@ -311,21 +312,30 @@ Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293).
 borgmatic includes a shell completion script (currently only for Bash) to
 support tab-completing borgmatic command-line actions and flags. Depending on
 how you installed borgmatic, this may be enabled by default. But if it's not,
-you can install the shell completion script globally:
+start by installing the `bash-completion` Linux package or the
+[`bash-completion@2`](https://formulae.brew.sh/formula/bash-completion@2)
+macOS Homebrew formula. Then, install the shell completion script globally:
+
+```bash
+sudo su -c "borgmatic --bash-completion > $(pkg-config --variable=completionsdir bash-completion)/borgmatic"
+```
+
+If you don't have `pkg-config` installed, you can try the following path
+instead:
 
 ```bash
 sudo su -c "borgmatic --bash-completion > /usr/share/bash-completion/completions/borgmatic"
 ```
 
-Alternatively, if you'd like to install the script for just the current user:
+Or, if you'd like to install the script for just the current user:
 
 ```bash
 mkdir --parents ~/.local/share/bash-completion/completions
 borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmatic
 ```
 
-In either case, you may also need to install the `bash-completion` Linux
+Finally, restart your shell (`exit` and open a new shell) so the completions
-package and restart your shell (`exit` and open a new shell).
+take effect.
 
 
 ### Colored output

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.6.2'
+VERSION = '1.6.3'
 
 
 setup(

tests/integration/commands/test_arguments.py

@@ -296,15 +296,6 @@ def test_parse_arguments_disallows_init_and_dry_run():
         )
 
 
-def test_parse_arguments_disallows_glob_archives_with_successful():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments(
-            '--config', 'myconfig', 'list', '--glob-archives', '*glob*', '--successful'
-        )
-
-
 def test_parse_arguments_disallows_repository_unless_action_consumes_it():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/unit/borg/test_list.py

@@ -1,3 +1,4 @@
+import argparse
 import logging
 
 import pytest
@@ -8,8 +9,6 @@ from borgmatic.borg import list as module
 from ..test_verbosity import insert_logging_mock
 
 BORG_LIST_LATEST_ARGUMENTS = (
-    '--glob-archives',
-    module.BORG_EXCLUDE_CHECKPOINTS_GLOB,
     '--last',
     '1',
     '--short',
@@ -108,156 +107,125 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
     )
 
 
-def test_list_archives_calls_borg_with_parameters():
+def test_make_list_command_includes_log_info():
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
-    )
-
-    module.list_archives(
-        repository='repo',
-        storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
-    )
-
-
-def test_list_archives_with_log_info_calls_borg_with_info_parameter():
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
-    )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(
+    command = module.make_list_command(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False),
     )
 
+    assert command == ('borg', 'list', '--info', 'repo')
 
-def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
+
-    flexmock(module).should_receive('execute_command').with_args(
+def test_make_list_command_includes_json_but_not_info():
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
-    )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(
+    command = module.make_list_command(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=True, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=True),
     )
 
+    assert command == ('borg', 'list', '--json', 'repo')
 
-def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
+
-    flexmock(module).should_receive('execute_command').with_args(
+def test_make_list_command_includes_log_debug():
-        ('borg', 'list', '--debug', '--show-rc', 'repo'),
-        output_log_level=logging.WARNING,
-        borg_local_path='borg',
-    )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(
+    command = module.make_list_command(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=False),
     )
 
+    assert command == ('borg', 'list', '--debug', '--show-rc', 'repo')
 
-def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
+
-    flexmock(module).should_receive('execute_command').with_args(
+def test_make_list_command_includes_json_but_not_debug():
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
-    )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(
+    command = module.make_list_command(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=True, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=True),
     )
 
-
+    assert command == ('borg', 'list', '--json', 'repo')
-def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
-    storage_config = {'lock_wait': 5}
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--lock-wait', '5', 'repo'),
-        output_log_level=logging.WARNING,
-        borg_local_path='borg',
-    )
-
-    module.list_archives(
-        repository='repo',
-        storage_config=storage_config,
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
-    )
 
 
-def test_list_archives_with_archive_calls_borg_with_archive_parameter():
+def test_make_list_command_includes_json():
-    storage_config = {}
+    command = module.make_list_command(
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
-    )
-
-    module.list_archives(
-        repository='repo',
-        storage_config=storage_config,
-        list_arguments=flexmock(archive='archive', paths=None, json=False, successful=False),
-    )
-
-
-def test_list_archives_with_path_calls_borg_with_path_parameter():
-    storage_config = {}
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo::archive', 'var/lib'),
-        output_log_level=logging.WARNING,
-        borg_local_path='borg',
-    )
-
-    module.list_archives(
-        repository='repo',
-        storage_config=storage_config,
-        list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False, successful=False),
-    )
-
-
-def test_list_archives_with_local_path_calls_borg_via_local_path():
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
-    )
-
-    module.list_archives(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
+        list_arguments=flexmock(archive=None, paths=None, json=True),
-        local_path='borg1',
     )
 
+    assert command == ('borg', 'list', '--json', 'repo')
 
-def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
+
-    flexmock(module).should_receive('execute_command').with_args(
+def test_make_list_command_includes_lock_wait():
-        ('borg', 'list', '--remote-path', 'borg1', 'repo'),
+    command = module.make_list_command(
-        output_log_level=logging.WARNING,
+        repository='repo',
-        borg_local_path='borg',
+        storage_config={'lock_wait': 5},
+        list_arguments=flexmock(archive=None, paths=None, json=False),
     )
 
-    module.list_archives(
+    assert command == ('borg', 'list', '--lock-wait', '5', 'repo')
+
+
+def test_make_list_command_includes_archive():
+    command = module.make_list_command(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False),
+        list_arguments=flexmock(archive='archive', paths=None, json=False),
-        remote_path='borg1',
     )
 
+    assert command == ('borg', 'list', 'repo::archive')
 
-def test_list_archives_with_short_calls_borg_with_short_parameter():
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--short', 'repo'),
-        output_log_level=logging.WARNING,
-        borg_local_path='borg',
-    ).and_return('[]')
 
-    module.list_archives(
+def test_make_list_command_includes_archive_and_path():
+    command = module.make_list_command(
         repository='repo',
         storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=False, short=True),
+        list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False),
     )
 
+    assert command == ('borg', 'list', 'repo::archive', 'var/lib')
+
+
+def test_make_list_command_includes_local_path():
+    command = module.make_list_command(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, paths=None, json=False),
+        local_path='borg2',
+    )
+
+    assert command == ('borg2', 'list', 'repo')
+
+
+def test_make_list_command_includes_remote_path():
+    command = module.make_list_command(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, paths=None, json=False),
+        remote_path='borg2',
+    )
+
+    assert command == ('borg', 'list', '--remote-path', 'borg2', 'repo')
+
+
+def test_make_list_command_includes_short():
+    command = module.make_list_command(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, paths=None, json=False, short=True),
+    )
+
+    assert command == ('borg', 'list', '--short', 'repo')
+
 
 @pytest.mark.parametrize(
     'argument_name',
@@ -273,45 +241,156 @@ def test_list_archives_with_short_calls_borg_with_short_parameter():
         'patterns_from',
     ),
 )
-def test_list_archives_passes_through_arguments_to_borg(argument_name):
+def test_make_list_command_includes_additional_flags(argument_name):
-    flexmock(module).should_receive('execute_command').with_args(
+    command = module.make_list_command(
-        ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
-        output_log_level=logging.WARNING,
-        borg_local_path='borg',
-    ).and_return('[]')
-
-    module.list_archives(
         repository='repo',
         storage_config={},
         list_arguments=flexmock(
-            archive=None, paths=None, json=False, successful=False, **{argument_name: 'value'}
+            archive=None,
+            paths=None,
+            json=False,
+            find_paths=None,
+            format=None,
+            **{argument_name: 'value'}
         ),
     )
 
+    assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo')
 
-def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
+
+def test_make_find_paths_considers_none_as_empty_paths():
+    assert module.make_find_paths(None) == ()
+
+
+def test_make_find_paths_passes_through_patterns():
+    find_paths = (
+        'fm:*',
+        'sh:**/*.txt',
+        're:^.*$',
+        'pp:root/somedir',
+        'pf:root/foo.txt',
+        'R /',
+        'r /',
+        'p /',
+        'P /',
+        '+ /',
+        '- /',
+        '! /',
+    )
+
+    assert module.make_find_paths(find_paths) == find_paths
+
+
+def test_make_find_paths_adds_globs_to_path_fragments():
+    assert module.make_find_paths(('foo.txt',)) == ('sh:**/*foo.txt*/**',)
+
+
+def test_list_archives_calls_borg_with_parameters():
+    list_arguments = argparse.Namespace(archive=None, paths=None, json=False, find_paths=None)
+
+    flexmock(module).should_receive('make_list_command').with_args(
+        repository='repo',
+        storage_config={},
+        list_arguments=list_arguments,
+        local_path='borg',
+        remote_path=None,
+    ).and_return(('borg', 'list', 'repo'))
+    flexmock(module).should_receive('make_find_paths').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--glob-archives', module.BORG_EXCLUDE_CHECKPOINTS_GLOB, 'repo'),
+        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
-        output_log_level=logging.WARNING,
+    ).once()
-        borg_local_path='borg',
-    ).and_return('[]')
 
     module.list_archives(
-        repository='repo',
+        repository='repo', storage_config={}, list_arguments=list_arguments,
-        storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=False, successful=True),
     )
 
 
-def test_list_archives_with_json_calls_borg_with_json_parameter():
+def test_list_archives_with_json_suppresses_most_borg_output():
+    list_arguments = argparse.Namespace(archive=None, paths=None, json=True, find_paths=None)
+
+    flexmock(module).should_receive('make_list_command').with_args(
+        repository='repo',
+        storage_config={},
+        list_arguments=list_arguments,
+        local_path='borg',
+        remote_path=None,
+    ).and_return(('borg', 'list', 'repo'))
+    flexmock(module).should_receive('make_find_paths').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
+        ('borg', 'list', 'repo'), output_log_level=None, borg_local_path='borg'
-    ).and_return('[]')
+    ).once()
 
-    json_output = module.list_archives(
+    module.list_archives(
-        repository='repo',
+        repository='repo', storage_config={}, list_arguments=list_arguments,
-        storage_config={},
-        list_arguments=flexmock(archive=None, paths=None, json=True, successful=False),
     )
 
-    assert json_output == '[]'
+
+def test_list_archives_calls_borg_with_local_path():
+    list_arguments = argparse.Namespace(archive=None, paths=None, json=False, find_paths=None)
+
+    flexmock(module).should_receive('make_list_command').with_args(
+        repository='repo',
+        storage_config={},
+        list_arguments=list_arguments,
+        local_path='borg2',
+        remote_path=None,
+    ).and_return(('borg2', 'list', 'repo'))
+    flexmock(module).should_receive('make_find_paths').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg2', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg2'
+    ).once()
+
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=list_arguments, local_path='borg2',
+    )
+
+
+def test_list_archives_calls_borg_multiple_times_with_find_paths():
+    glob_paths = ('**/*foo.txt*/**',)
+    list_arguments = argparse.Namespace(
+        archive=None, paths=None, json=False, find_paths=['foo.txt'], format=None
+    )
+
+    flexmock(module).should_receive('make_list_command').and_return(
+        ('borg', 'list', 'repo')
+    ).and_return(('borg', 'list', 'repo::archive1')).and_return(('borg', 'list', 'repo::archive2'))
+    flexmock(module).should_receive('make_find_paths').and_return(glob_paths)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo'), output_log_level=None, borg_local_path='borg'
+    ).and_return(
+        'archive1   Sun, 2022-05-29 15:27:04 [abc]\narchive2   Mon, 2022-05-30 19:47:15 [xyz]'
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo::archive1') + glob_paths,
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo::archive2') + glob_paths,
+        output_log_level=logging.WARNING,
+        borg_local_path='borg',
+    ).once()
+
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=list_arguments,
+    )
+
+
+def test_list_archives_calls_borg_with_archive():
+    list_arguments = argparse.Namespace(archive='archive', paths=None, json=False, find_paths=None)
+
+    flexmock(module).should_receive('make_list_command').with_args(
+        repository='repo',
+        storage_config={},
+        list_arguments=list_arguments,
+        local_path='borg',
+        remote_path=None,
+    ).and_return(('borg', 'list', 'repo::archive'))
+    flexmock(module).should_receive('make_find_paths').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
+    ).once()
+
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=list_arguments,
+    )

tests/unit/hooks/test_healthchecks.py

@@ -69,10 +69,18 @@ def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload
     assert payload == ''
 
 
+def mock_logger():
+    logger = flexmock()
+    logger.should_receive('addHandler')
+    logger.should_receive('removeHandler')
+    flexmock(module.logging).should_receive('getLogger').and_return(logger)
+
+
 def test_initialize_monitor_creates_log_handler_with_ping_body_limit():
     ping_body_limit = 100
     monitoring_log_level = 1
 
+    mock_logger()
     flexmock(module).should_receive('Forgetful_buffering_handler').with_args(
         ping_body_limit - len(module.PAYLOAD_TRUNCATION_INDICATOR), monitoring_log_level
     ).once()
@@ -85,6 +93,7 @@ def test_initialize_monitor_creates_log_handler_with_ping_body_limit():
 def test_initialize_monitor_creates_log_handler_with_default_ping_body_limit():
     monitoring_log_level = 1
 
+    mock_logger()
     flexmock(module).should_receive('Forgetful_buffering_handler').with_args(
         module.DEFAULT_PING_BODY_LIMIT_BYTES - len(module.PAYLOAD_TRUNCATION_INDICATOR),
         monitoring_log_level,
@@ -97,6 +106,7 @@ def test_initialize_monitor_creates_log_handler_with_zero_ping_body_limit():
     ping_body_limit = 0
     monitoring_log_level = 1
 
+    mock_logger()
     flexmock(module).should_receive('Forgetful_buffering_handler').with_args(
         ping_body_limit, monitoring_log_level
     ).once()
@@ -107,6 +117,7 @@ def test_initialize_monitor_creates_log_handler_with_zero_ping_body_limit():
 
 
 def test_initialize_monitor_creates_log_handler_when_send_logs_true():
+    mock_logger()
     flexmock(module).should_receive('Forgetful_buffering_handler').once()
 
     module.initialize_monitor(
@@ -115,6 +126,7 @@ def test_initialize_monitor_creates_log_handler_when_send_logs_true():
 
 
 def test_initialize_monitor_bails_when_send_logs_false():
+    mock_logger()
     flexmock(module).should_receive('Forgetful_buffering_handler').never()
 
     module.initialize_monitor(

tests/unit/hooks/test_ntfy.py

@@ -0,0 +1,135 @@
+from enum import Enum
+
+from flexmock import flexmock
+
+from borgmatic.hooks import ntfy as module
+
+default_base_url = 'https://ntfy.sh'
+custom_base_url = 'https://ntfy.example.com'
+topic = 'borgmatic-unit-testing'
+
+custom_message_config = {
+    'title': 'Borgmatic unit testing',
+    'message': 'Borgmatic unit testing',
+    'priority': 'min',
+    'tags': '+1',
+}
+
+custom_message_headers = {
+    'X-Title': custom_message_config['title'],
+    'X-Message': custom_message_config['message'],
+    'X-Priority': custom_message_config['priority'],
+    'X-Tags': custom_message_config['tags'],
+}
+
+
+def return_default_message_headers(state=Enum):
+    headers = {
+        'X-Title': f'A Borgmatic {state.name} event happened',
+        'X-Message': f'A Borgmatic {state.name} event happened',
+        'X-Priority': 'default',
+        'X-Tags': 'borgmatic',
+    }
+    return headers
+
+
+def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.FAIL),
+    ).once()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+    )
+
+
+def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').never()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').never()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.FINISH,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail():
+    hook_config = {'topic': topic, 'server': custom_base_url}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{custom_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.FAIL),
+    ).once()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+    )
+
+
+def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').never()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True
+    )
+
+
+def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail():
+    hook_config = {'topic': topic, 'fail': custom_message_config}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}', headers=custom_message_headers,
+    ).once()
+
+    module.ping_monitor(
+        hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
+    )
+
+
+def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start():
+    hook_config = {'topic': topic, 'states': ['start', 'fail']}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.START),
+    ).once()
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )
+
+
+def test_ping_monitor_with_connection_error_does_not_raise():
+    hook_config = {'topic': topic}
+    flexmock(module.requests).should_receive('post').with_args(
+        f'{default_base_url}/{topic}',
+        headers=return_default_message_headers(module.monitor.State.FAIL),
+    ).and_raise(module.requests.exceptions.ConnectionError)
+
+    module.ping_monitor(
+        hook_config,
+        'config.yaml',
+        module.monitor.State.FAIL,
+        monitoring_log_level=1,
+        dry_run=False,
+    )