Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1293
Reviewed-by: Dan Helfman <witten@torsion.org>

NEWS

@@ -1,3 +1,14 @@
+2.1.5
+ * #1229: Document the permissions needed for the PostgreSQL database hook:
+   https://torsion.org/borgmatic/reference/configuration/data-sources/postgresql/
+ * #1289: Add mutual TLS support for the Loki monitoring hook. See the documentation for more
+   information: https://torsion.org/borgmatic/reference/configuration/monitoring/loki/
+ * #1292: Fix a "source directories do not exist" regression when configuration paths are relative
+   symlinks and the bootstrap data source hook is enabled.
+ * #1294: Fix a regression in which SSH warnings from remote repositories broke the "spot" check
+   and other actions as well.
+ * #1295: Fix the ZFS hook to properly unmount snapshots for empty datasets.
+
 2.1.4
  * #1266: Add a stand-alone borgmatic Linux binary to the release downloads to serve as another way
    to install borgmatic. Consider this binary a beta feature.
@@ -10,7 +21,7 @@
  * #1268: Fix the "spot" check, "extract" check, and all uses of the archive name "latest" to
    respect the "match_archives" and "archives_name_format" options. This means that borgmatic now
    uses the "latest" archive that also matches those options instead of the latest archive overall.
- * When Borg exits with a warning exit code, show a description of it, so you don't have to lookup
+ * When Borg exits with a warning exit code, show a description of it, so you don't have to look up
    the code.
  * Split out borgmatic installation documentation to its own page, so it's easier to find.
  * Switch the default borgmatic installation method from pipx to uv, as uv is faster and used for

borgmatic/config/schema.yaml

@@ -3098,6 +3098,24 @@ properties:
                     Grafana Loki log URL to notify when a backup begins,
                     ends, or fails.
                 example: "http://localhost:3100/loki/api/v1/push"
+            tls:
+                type: object
+                additionalProperties: false
+                properties:
+                    cert_path:
+                        type: string
+                        description: |
+                            Path to a PEM client certificate file for mutual
+                            TLS authentication.
+                        example: /etc/borgmatic/loki-client.crt
+                    key_path:
+                        type: string
+                        description: |
+                            Path to a PEM private key file for the client
+                            certificate.
+                        example: /etc/borgmatic/loki-client.key
+                description: |
+                    TLS options for mutual TLS (mTLS) authentication with Loki.
             labels:
                 type: object
                 additionalProperties:

borgmatic/execute.py

@@ -315,8 +315,8 @@ def log_buffer_lines(
     Given a dict from buffer object to Buffer_reader, a dict from subprocess.Popen() instance to
     Process_metadata instance, a requested output log level for stdout, Borg's local path, and
     whether to capture stderr, read and log any ready output lines from the buffers. Additionally,
-    for any log records with a log level that meets or exceeds the output log level, yield those log
-    messages for capture.
+    for any log records with a log level the same as the output log level, yield those log messages
+    for capture.
 
     This function just does one "turn of the crank" of logging buffer output. It is intended to be
     called repeatedly to continue to process buffers.
@@ -367,9 +367,7 @@ def log_buffer_lines(
             )
 
             if (
-                log_record.levelno is None
-                or output_log_level is None
-                or log_record.levelno >= output_log_level
+                log_record.levelno is None or log_record.levelno == output_log_level
             ) and process_metadatas[reader.process].capture:
                 yield log_record.getMessage()
 
@@ -429,8 +427,8 @@ def log_remaining_buffer_lines(
     Given a dict from buffer object to Buffer_reader, a dict from subprocess.Popen() instance to
     Process_metadata instance, a requested output log level for stdout, Borg's local path, and
     whether to capture stderr, drain and log any remaining output lines from the buffers until
-    they're empty. Additionally, for any log records with a log level that meets or exceeds the
-    output log level, yield those log messages for capture.
+    they're empty. Additionally, for any log records with a log level the same as the output log
+    level, yield those log messages for capture.
     '''
     for output_buffer, reader in buffer_readers.items():
         if not reader.process:
@@ -451,9 +449,7 @@ def log_remaining_buffer_lines(
                 )
 
                 if (
-                    log_record.levelno is None
-                    or output_log_level is None
-                    or log_record.levelno >= output_log_level
+                    log_record.levelno is None or log_record.levelno == output_log_level
                 ) and process_metadatas[reader.process].capture:
                     yield log_record.getMessage()
 

borgmatic/hooks/data_source/bootstrap.py

@@ -28,17 +28,20 @@ def resolve_config_path_symlinks(path):
     Given a path, resolve and yield each successive symlink until the final non-symlink target. If
     the given path isn't a symlink, then just yield it.
 
+    The purpose of this is to ensure that configuration files that are behind a symbolic link (or
+    several) actually get backed up.
+
     Raise ValueError if we have to follow too many symlinks without getting to the final target.
     '''
-    original_path = path
+    original_path = os.path.normpath(path)
 
     for _ in range(MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW):
-        yield os.path.abspath(path)
+        yield path
 
         if not os.path.islink(path):
             return
 
-        path = os.readlink(path)
+        path = os.path.normpath(os.path.join(os.path.dirname(path), os.readlink(path)))
 
     raise ValueError(f'Too many symlinks to follow for configuration path: {original_path}')
 

borgmatic/hooks/data_source/zfs.py

@@ -2,6 +2,7 @@ import collections
 import glob
 import hashlib
 import logging
+import operator
 import os
 import shutil
 import subprocess
@@ -123,7 +124,8 @@ def get_datasets_to_backup(zfs_command, patterns):
 
 def get_all_dataset_mount_points(zfs_command):
     '''
-    Given a ZFS command to run, return all ZFS datasets as a sequence of sorted mount points.
+    Given a ZFS command to run, return a dict from ZFS dataset name to mount point (reverse sorted
+    by mount point).
     '''
     list_lines = borgmatic.execute.execute_command_and_capture_output(
         (
@@ -133,19 +135,23 @@ def get_all_dataset_mount_points(zfs_command):
             '-t',
             'filesystem',
             '-o',
-            'mountpoint',
+            'name,mountpoint',
         ),
         close_fds=True,
     )
 
-    return tuple(
+    return dict(
         sorted(
-            {
-                mount_point
+            (
+                (dataset_name, mount_point)
                 for line in list_lines
-                for mount_point in (line.rstrip(),)
+                for (dataset_name, mount_point) in (line.rstrip().split('\t'),)
                 if mount_point != 'none'
-            },
+            ),
+            key=operator.itemgetter(1),
+            # Reversing the sorted datasets ensures that we unmount the longer mount point paths of
+            # child datasets before the shorter mount point paths of parent datasets.
+            reverse=True,
         ),
     )
 
@@ -376,7 +382,8 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, p
     zfs_command = hook_config.get('zfs_command', 'zfs')
 
     try:
-        dataset_mount_points = get_all_dataset_mount_points(zfs_command)
+        dataset_name_to_mount_point = get_all_dataset_mount_points(zfs_command)
+        full_snapshot_names = get_all_snapshots(zfs_command)
     except FileNotFoundError:
         logger.debug(f'Could not find "{zfs_command}" command')
         return
@@ -393,19 +400,20 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, p
     )
     logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}')
     umount_command = hook_config.get('umount_command', 'umount')
+    snapshot_dataset_names = {
+        full_snapshot_name.split('@')[0] for full_snapshot_name in full_snapshot_names
+    }
 
     for snapshots_directory in glob.glob(snapshots_glob):
         if not os.path.isdir(snapshots_directory):
             continue
 
-        # Reversing the sorted datasets ensures that we unmount the longer mount point paths of
-        # child datasets before the shorter mount point paths of parent datasets.
-        for mount_point in reversed(dataset_mount_points):
+        for dataset_name, mount_point in dataset_name_to_mount_point.items():
             snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep))
 
-            # If the snapshot mount path is empty, this is probably just a "shadow" of a nested
-            # dataset and therefore there's nothing to unmount.
-            if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path):
+            # If this dataset name does not correspond to a known snapshot, then this is probably
+            # just a "shadow" of a nested dataset and therefore there's nothing to unmount.
+            if not os.path.isdir(snapshot_mount_path) or dataset_name not in snapshot_dataset_names:
                 continue
 
             # This might fail if the path is already mounted, but we swallow errors here since we'll
@@ -435,8 +443,6 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, p
             shutil.rmtree(snapshot_mount_path, ignore_errors=True)
 
     # Destroy snapshots.
-    full_snapshot_names = get_all_snapshots(zfs_command)
-
     for full_snapshot_name in full_snapshot_names:
         # Only destroy snapshots that borgmatic actually created!
         if not full_snapshot_name.split('@')[-1].startswith(BORGMATIC_SNAPSHOT_PREFIX):

borgmatic/hooks/monitoring/loki.py

@@ -27,9 +27,16 @@ class Loki_log_buffer:
     adding labels to the log stream and takes care of communication with Loki.
     '''
 
-    def __init__(self, url, dry_run):
+    def __init__(self, url, dry_run, tls_cert_path=None, tls_key_path=None):
+        '''
+        Given a Loki URL, a dry run flag, and optional TLS certificate and key paths for mTLS authentication,
+        create an instance of Loki_log_buffer.
+        '''
+
         self.url = url
         self.dry_run = dry_run
+        self.tls_cert_path = tls_cert_path
+        self.tls_key_path = tls_key_path
         self.root = {'streams': [{'stream': {}, 'values': []}]}
 
     def add_value(self, value):
@@ -77,6 +84,7 @@ class Loki_log_buffer:
                     'Content-Type': 'application/json',
                     'User-Agent': 'borgmatic',
                 },
+                cert=(self.tls_cert_path, self.tls_key_path) if self.tls_cert_path else None,
             )
             result.raise_for_status()
         except requests.RequestException:
@@ -88,7 +96,7 @@ class Loki_log_handler(logging.Handler):
     A log handler that sends logs to Loki.
     '''
 
-    def __init__(self, url, send_logs, log_level, dry_run):
+    def __init__(self, url, send_logs, log_level, dry_run, tls_cert_path=None, tls_key_path=None):
         '''
         Given a URL to send logs to, whether all borgmatic logs should be sent (or just explicitly
         added messages from this hook), the log level to use (influencing which logs get sent), and
@@ -96,7 +104,9 @@ class Loki_log_handler(logging.Handler):
         '''
         super().__init__()
 
-        self.buffer = Loki_log_buffer(url, dry_run)
+        self.buffer = Loki_log_buffer(
+            url, dry_run, tls_cert_path=tls_cert_path, tls_key_path=tls_key_path
+        )
         self.send_logs = send_logs
         self.setLevel(log_level)
 
@@ -139,7 +149,21 @@ def initialize_monitor(hook_config, config, config_filename, monitoring_log_leve
     Add a handler to the root logger to regularly send the logs to Loki.
     '''
     url = hook_config.get('url')
-    loki = Loki_log_handler(url, hook_config.get('send_logs', False), monitoring_log_level, dry_run)
+    tls = hook_config.get('tls', {})
+
+    if bool(tls.get('cert_path')) != bool(tls.get('key_path')):
+        raise ValueError(
+            'Invalid Loki TLS configuration: cert_path and key_path must both be set or both be unset'
+        )
+
+    loki = Loki_log_handler(
+        url,
+        hook_config.get('send_logs', False),
+        monitoring_log_level,
+        dry_run,
+        tls_cert_path=tls.get('cert_path'),
+        tls_key_path=tls.get('key_path'),
+    )
 
     for key, value in hook_config.get('labels').items():
         if value == '__hostname':

docs/how-to/install-borgmatic.md

@@ -80,6 +80,7 @@ installing borgmatic:
 
  * [container image with scheduled backups](https://github.com/borgmatic-collective/docker-borgmatic) (+ Docker Compose files)
  * [container image with multi-arch and Docker CLI support](https://github.com/modem7/docker-borgmatic)
+ * [Borgmatic Director UI](https://github.com/SpeedbitsInfinityTools/borgmatic-ui-community)
 
 #### Operating system packages
 

docs/reference/configuration/data-sources/postgresql.md

@@ -13,6 +13,61 @@ postgresql_databases:
     - name: users
 ```
 
+See below for the full set of configuration options available, including
+hostname, PostgreSQL username, password, etc.
+
+
+## Permissions
+
+### Dumping
+
+In order to dump your database as part of creating a backup, the PostgreSQL user
+performing the dump needs relevant permissions. A common way to accomplish this
+is to connect as the PostgreSQL superuser, usually `postgres`. However, if you'd
+like to connect as a non-superuser, that user will need permissions to:
+
+ * connect to the database
+ * read tables and sequences
+
+Here is one way to do that with PostgreSQL 14+:
+
+```sql
+GRANT CONNECT ON DATABASE example_database TO database_user;
+GRANT pg_read_all_data TO database_user;
+```
+
+And here is an alternate way to accomplish something similar that limits read access
+to a particular schema instead of the whole cluster. Replace "public" with the
+name of the schema you're using:
+
+```sql
+GRANT CONNECT ON DATABASE example_database TO database_user;
+GRANT USAGE ON SCHEMA public TO database_user;
+-- Grant read privileges on all current and future tables in the schema.
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO database_user;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO database_user;
+-- Grant read privileges on all current and future indexes in the schema.
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO database_user;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO database_user;
+```
+
+### Restoring
+
+If you also want this user to be able to restore your database, and you're not
+restoring as the PostgreSQL superuser, then you'll need to grant write and
+`ANALYZE` permissions as well. For instance:
+
+```sql
+GRANT pg_write_all_data TO database_user;
+GRANT pg_maintain TO database_user;
+```
+
+Or you can perform schema-level grants if you prefer.
+
+For more information, see the PostgreSQL documentation on [PostgreSQL predefined
+roles](https://www.postgresql.org/docs/current/predefined-roles.html) and
+[privileges](https://www.postgresql.org/docs/current/ddl-priv.html).
+
 
 ## Full configuration
 

docs/reference/configuration/monitoring/loki.md

@@ -89,3 +89,34 @@ for more information.
 <span class="minilink minilink-addedin">New in version 2.0.0</span>Set the
 defaults for these flags in your borgmatic configuration via the
 `monitoring_verbosity`, `list`, and `statistics` options.
+
+
+### Mutual TLS authentication
+
+<span class="minilink minilink-addedin">New in version 2.1.5</span> Since Loki
+does not come with a built-in authentication layer
+[(doc)](https://grafana.com/docs/loki/latest/operations/authentication/), this
+feature is typically used alongside a reverse proxy (such as
+[nginx](https://docs.nginx.com/waf/configure/secure-mtls/) or
+[Traefik](https://doc.traefik.io/traefik/reference/routing-configuration/http/tls/tls-options/#client-authentication-mtls))
+that handles mTLS termination.
+
+If your setup is configured for mTLS authentication, you can provide a client
+certificate and private key:
+
+```yaml
+loki:
+    url: https://loki.fqdn/loki/api/v1/push
+
+    labels:
+        app: borgmatic
+
+    tls:
+        cert_path: /etc/borgmatic/loki-client.crt
+        key_path: /etc/borgmatic/loki-client.key
+```
+
+Both `cert_path` and `key_path` must be
+[PEM-encoded](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail). They are
+passed directly to the underlying HTTP client, so the standard mutual TLS
+handshake is performed for every request borgmatic sends to Loki.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "2.1.4"
+version = "2.1.5"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]

tests/integration/hooks/monitoring/test_loki.py

@@ -90,7 +90,7 @@ def test_ping_monitor_sends_log_message():
     config_filename = 'test.yaml'
     post_called = False
 
-    def post(url, data, timeout, headers):
+    def post(url, data, timeout, headers, **kwargs):
         nonlocal post_called
         post_called = True
 

tests/integration/test_execute.py

@@ -137,15 +137,18 @@ def test_log_outputs_logs_stderr_as_error():
         (),
     ).and_return((echo_process.stdout, echo_process.stderr))
 
-    assert tuple(
-        module.log_outputs(
-            (echo_process,),
-            exclude_stdouts=(),
-            output_log_level=logging.INFO,
-            borg_local_path='borg',
-            borg_exit_codes=None,
+    assert (
+        tuple(
+            module.log_outputs(
+                (echo_process,),
+                exclude_stdouts=(),
+                output_log_level=logging.INFO,
+                borg_local_path='borg',
+                borg_exit_codes=None,
+            )
         )
-    ) == ('error',)
+        == ()
+    )
 
 
 def test_log_outputs_skips_logs_for_process_with_none_stdout():

tests/unit/hooks/data_source/test_bootstrap.py

@@ -7,14 +7,12 @@ from borgmatic.hooks.data_source import bootstrap as module
 
 
 def test_resolve_config_path_symlinks_passes_through_non_symlink():
-    flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
     flexmock(module.os.path).should_receive('islink').and_return(False)
 
     assert tuple(module.resolve_config_path_symlinks('test.yaml')) == ('test.yaml',)
 
 
 def test_resolve_config_path_symlinks_follows_each_symlink():
-    flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
     flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
     flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
     flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(False)
@@ -29,9 +27,29 @@ def test_resolve_config_path_symlinks_follows_each_symlink():
     )
 
 
+def test_resolve_config_path_symlinks_follows_each_relative_symlink():
+    flexmock(module.os.path).should_receive('islink').with_args('foo/bar/test.yaml').and_return(
+        True
+    )
+    flexmock(module.os.path).should_receive('islink').with_args('foo/dest1.yaml').and_return(True)
+    flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(False)
+    flexmock(module.os).should_receive('readlink').with_args('foo/bar/test.yaml').and_return(
+        '../dest1.yaml'
+    )
+    flexmock(module.os).should_receive('readlink').with_args('foo/dest1.yaml').and_return(
+        '../dest2.yaml'
+    )
+    flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').never()
+
+    assert tuple(module.resolve_config_path_symlinks('foo/bar/test.yaml')) == (
+        'foo/bar/test.yaml',
+        'foo/dest1.yaml',
+        'dest2.yaml',
+    )
+
+
 def test_resolve_config_path_symlinks_with_too_many_symlinks_raises():
     flexmock(module).MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 2
-    flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
     flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
     flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
     flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(True)

tests/unit/hooks/data_source/test_zfs.py

@@ -223,40 +223,34 @@ def test_get_datasets_to_backup_with_invalid_list_output_raises():
         module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar')))
 
 
-def test_get_all_dataset_mount_points_omits_none():
+def test_get_all_dataset_mount_points_omits_none_and_reverse_orders_by_mount_path():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output',
     ).and_yield(
-        '/dataset',
-        'none',
-        '/other',
+        'dataset\t/path',
+        'thing\tnone',
+        'other\t/other',
     )
-    flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_patterns',
-    ).and_return((Pattern('/dataset'),))
 
-    assert module.get_all_dataset_mount_points('zfs') == (
-        ('/dataset'),
-        ('/other'),
+    assert tuple(module.get_all_dataset_mount_points('zfs').items()) == (
+        ('dataset', '/path'),
+        ('other', '/other'),
     )
 
 
-def test_get_all_dataset_mount_points_omits_duplicates():
+def test_get_all_dataset_mount_points_omits_duplicates_and_reverse_orders_by_mount_path():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output',
     ).and_return(
-        '/dataset',
-        '/other',
-        '/dataset',
-        '/other',
+        'dataset\t/path',
+        'other\t/other',
+        'dataset\t/path',
+        'other\t/other',
     )
-    flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
-        'get_contained_patterns',
-    ).and_return((Pattern('/dataset'),))
 
-    assert module.get_all_dataset_mount_points('zfs') == (
-        ('/dataset'),
-        ('/other'),
+    assert tuple(module.get_all_dataset_mount_points('zfs').items()) == (
+        ('dataset', '/path'),
+        ('other', '/other'),
     )
 
 
@@ -525,7 +519,12 @@ def test_get_all_snapshots_parses_list_output():
 
 
 def test_remove_data_source_dumps_unmounts_and_destroys_snapshots():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -533,15 +532,11 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots():
         lambda path: [path.replace('*', 'b33f')],
     )
     flexmock(module.os.path).should_receive('isdir').and_return(True)
-    flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
     flexmock(module.shutil).should_receive('rmtree')
     flexmock(module).should_receive('unmount_snapshot').with_args(
         'umount',
         '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
     ).once()
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
     flexmock(module).should_receive('destroy_snapshot').with_args(
         'zfs',
         'dataset@borgmatic-1234',
@@ -557,7 +552,12 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots():
 
 
 def test_remove_data_source_dumps_use_custom_commands():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -565,15 +565,11 @@ def test_remove_data_source_dumps_use_custom_commands():
         lambda path: [path.replace('*', 'b33f')],
     )
     flexmock(module.os.path).should_receive('isdir').and_return(True)
-    flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
     flexmock(module.shutil).should_receive('rmtree')
     flexmock(module).should_receive('unmount_snapshot').with_args(
         '/usr/local/bin/umount',
         '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
     ).once()
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
     flexmock(module).should_receive('destroy_snapshot').with_args(
         '/usr/local/bin/zfs',
         'dataset@borgmatic-1234',
@@ -639,7 +635,12 @@ def test_remove_data_source_dumps_bails_for_zfs_command_error():
 
 
 def test_remove_data_source_dumps_bails_for_missing_umount_command():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -647,13 +648,11 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command():
         lambda path: [path.replace('*', 'b33f')],
     )
     flexmock(module.os.path).should_receive('isdir').and_return(True)
-    flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
     flexmock(module.shutil).should_receive('rmtree')
     flexmock(module).should_receive('unmount_snapshot').with_args(
         '/usr/local/bin/umount',
         '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
     ).and_raise(FileNotFoundError)
-    flexmock(module).should_receive('get_all_snapshots').never()
     flexmock(module).should_receive('destroy_snapshot').never()
     hook_config = {'zfs_command': '/usr/local/bin/zfs', 'umount_command': '/usr/local/bin/umount'}
 
@@ -667,7 +666,12 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command():
 
 
 def test_remove_data_source_dumps_swallows_umount_command_error():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -675,15 +679,11 @@ def test_remove_data_source_dumps_swallows_umount_command_error():
         lambda path: [path.replace('*', 'b33f')],
     )
     flexmock(module.os.path).should_receive('isdir').and_return(True)
-    flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
     flexmock(module.shutil).should_receive('rmtree')
     flexmock(module).should_receive('unmount_snapshot').with_args(
         '/usr/local/bin/umount',
         '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
     ).and_raise(module.subprocess.CalledProcessError(1, 'wtf'))
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
     flexmock(module).should_receive('destroy_snapshot').with_args(
         '/usr/local/bin/zfs',
         'dataset@borgmatic-1234',
@@ -700,7 +700,12 @@ def test_remove_data_source_dumps_swallows_umount_command_error():
 
 
 def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_not_actually_directories():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -710,9 +715,6 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_no
     flexmock(module.os.path).should_receive('isdir').and_return(False)
     flexmock(module.shutil).should_receive('rmtree').never()
     flexmock(module).should_receive('unmount_snapshot').never()
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
     flexmock(module).should_receive('destroy_snapshot').with_args(
         'zfs',
         'dataset@borgmatic-1234',
@@ -728,7 +730,12 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_no
 
 
 def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_not_actually_directories():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -741,12 +748,8 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no
     flexmock(module.os.path).should_receive('isdir').with_args(
         '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
     ).and_return(False)
-    flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
     flexmock(module.shutil).should_receive('rmtree')
     flexmock(module).should_receive('unmount_snapshot').never()
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
     flexmock(module).should_receive('destroy_snapshot').with_args(
         'zfs',
         'dataset@borgmatic-1234',
@@ -761,8 +764,13 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no
     )
 
 
-def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_empty():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_for_unknown_shapshots():
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset', 'sub': '/mnt/dataset/shadow'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -775,14 +783,16 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_em
     flexmock(module.os.path).should_receive('isdir').with_args(
         '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
     ).and_return(True)
-    flexmock(module.os).should_receive('listdir').with_args(
-        '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
-    ).and_return([])
+    flexmock(module.os.path).should_receive('isdir').with_args(
+        '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset/shadow',
+    ).and_return(True)
     flexmock(module.shutil).should_receive('rmtree')
-    flexmock(module).should_receive('unmount_snapshot').never()
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
+    flexmock(module).should_receive('unmount_snapshot').with_args(
+        'umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset'
+    ).once()
+    flexmock(module).should_receive('unmount_snapshot').with_args(
+        'umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset/shadow'
+    ).never()
     flexmock(module).should_receive('destroy_snapshot').with_args(
         'zfs',
         'dataset@borgmatic-1234',
@@ -798,7 +808,12 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_em
 
 
 def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtree_succeeds():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -811,12 +826,8 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre
     flexmock(module.os.path).should_receive('isdir').with_args(
         '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset',
     ).and_return(True).and_return(False)
-    flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
     flexmock(module.shutil).should_receive('rmtree')
     flexmock(module).should_receive('unmount_snapshot').never()
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
     flexmock(module).should_receive('destroy_snapshot').with_args(
         'zfs',
         'dataset@borgmatic-1234',
@@ -832,7 +843,12 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre
 
 
 def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy():
-    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',))
+    flexmock(module).should_receive('get_all_dataset_mount_points').and_return(
+        {'dataset': '/mnt/dataset'}
+    )
+    flexmock(module).should_receive('get_all_snapshots').and_return(
+        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
+    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'replace_temporary_subdirectory_with_glob',
     ).and_return('/run/borgmatic')
@@ -840,12 +856,8 @@ def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy():
         lambda path: [path.replace('*', 'b33f')],
     )
     flexmock(module.os.path).should_receive('isdir').and_return(True)
-    flexmock(module.os).should_receive('listdir').and_return(['file.txt'])
     flexmock(module.shutil).should_receive('rmtree').never()
     flexmock(module).should_receive('unmount_snapshot').never()
-    flexmock(module).should_receive('get_all_snapshots').and_return(
-        ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid'),
-    )
     flexmock(module).should_receive('destroy_snapshot').never()
 
     module.remove_data_source_dumps(

tests/unit/hooks/monitoring/test_loki.py

@@ -1,5 +1,6 @@
 import json
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks.monitoring import loki as module
@@ -78,3 +79,49 @@ def test_loki_log_handler_flush_with_empty_buffer_does_not_raise():
     handler = module.Loki_log_handler(flexmock(), send_logs=False, log_level=10, dry_run=False)
 
     handler.flush()
+
+
+def test_loki_log_buffer_init_with_tls_stores_cert_and_key_paths():
+    buffer = module.Loki_log_buffer(
+        flexmock(),
+        dry_run=False,
+        tls_cert_path='/path/to/cert.crt',
+        tls_key_path='/path/to/key.key',
+    )
+
+    assert buffer.tls_cert_path == '/path/to/cert.crt'
+    assert buffer.tls_key_path == '/path/to/key.key'
+
+
+def test_loki_log_handler_init_with_tls_passes_paths_to_buffer():
+    handler = module.Loki_log_handler(
+        flexmock(),
+        send_logs=False,
+        log_level=10,
+        dry_run=False,
+        tls_cert_path='/path/to/cert.crt',
+        tls_key_path='/path/to/key.key',
+    )
+
+    assert handler.buffer.tls_cert_path == '/path/to/cert.crt'
+    assert handler.buffer.tls_key_path == '/path/to/key.key'
+
+
+def test_initialize_monitor_with_only_cert_path_raises():
+    hook_config = {
+        'url': 'http://localhost:3100/loki/api/v1/push',
+        'tls': {'cert_path': '/path/to/cert.crt'},
+    }
+
+    with pytest.raises(ValueError):
+        module.initialize_monitor(hook_config, {}, 'test.yaml', 1, False)
+
+
+def test_initialize_monitor_with_only_key_path_raises():
+    hook_config = {
+        'url': 'http://localhost:3100/loki/api/v1/push',
+        'tls': {'key_path': '/path/to/key.key'},
+    }
+
+    with pytest.raises(ValueError):
+        module.initialize_monitor(hook_config, {}, 'test.yaml', 1, False)

tests/unit/test_execute.py

@@ -486,7 +486,7 @@ def test_log_buffer_lines_with_ready_buffer_and_higher_log_level_and_capture_pro
     )
 
 
-def test_log_buffer_lines_with_ready_buffer_and_lower_log_level_and_capture_process_yields_each_line():
+def test_log_buffer_lines_with_ready_buffer_and_log_level_equal_to_output_log_level_and_capture_process_yields_each_line():
     process = flexmock(poll=lambda: None, stderr=flexmock(), args=flexmock())
     buffer_readers = {
         flexmock(): module.Buffer_reader(lines=iter((('hi', 'there'),)), process=process)
@@ -497,7 +497,7 @@ def test_log_buffer_lines_with_ready_buffer_and_lower_log_level_and_capture_proc
     ).and_return(list(buffer_readers.keys()), [], [])
     flexmock(module).should_receive('parse_log_line').and_return(flexmock())
     flexmock(module).should_receive('handle_log_record').and_return(
-        flexmock(levelno=module.logging.ERROR, getMessage=lambda: 'message')
+        flexmock(levelno=module.logging.INFO, getMessage=lambda: 'message')
     ).twice()
 
     assert tuple(
@@ -1256,7 +1256,7 @@ def test_log_remaining_buffer_lines_with_higher_log_level_and_capture_process_do
     )
 
 
-def test_log_remaining_buffer_lines_with_lower_log_level_and_capture_process_does_yields_each_line():
+def test_log_remaining_buffer_lines_with_log_level_equal_to_output_log_level_and_capture_process_yields_each_line():
     process = flexmock(stderr=flexmock(), args=flexmock())
     buffer_readers = {
         flexmock(): module.Buffer_reader(
@@ -1269,7 +1269,7 @@ def test_log_remaining_buffer_lines_with_lower_log_level_and_capture_process_doe
         line=str, log_level=object, elevate_stderr=False, borg_local_path=object, command=object
     ).and_return(flexmock()).twice()
     flexmock(module).should_receive('handle_log_record').and_return(
-        flexmock(levelno=module.logging.ERROR, getMessage=lambda: 'message')
+        flexmock(levelno=module.logging.INFO, getMessage=lambda: 'message')
     ).twice()
 
     assert tuple(