Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/952

NEWS

@@ -1,3 +1,13 @@
+1.9.5
+ * #418: Backup and restore databases that have the same name but with different ports, hostnames,
+   or hooks.
+ * #947: To avoid a hang in the database hooks, error and exit when the borgmatic runtime
+   directory overlaps with the configured excludes.
+ * #954: Fix a findmnt command error in the Btrfs hook by switching to parsing JSON output.
+ * #956: Fix the printing of a color reset code even when color is disabled.
+ * #958: Drop colorama as a library dependency.
+ * When the ZFS, Btrfs, or LVM hooks aren't configured, don't try to cleanup snapshots for them.
+
 1.9.4
  * #80 (beta): Add an LVM hook for snapshotting and backing up LVM logical volumes. See the
    documentation for more information:

borgmatic/actions/restore.py

@@ -1,4 +1,4 @@
-import copy
+import collections
 import logging
 import os
 import pathlib
@@ -17,53 +17,90 @@ import borgmatic.hooks.dispatch
 logger = logging.getLogger(__name__)
 
 
-UNSPECIFIED_HOOK = object()
+UNSPECIFIED = object()
 
 
-def get_configured_data_source(
-    config,
-    archive_data_source_names,
-    hook_name,
-    data_source_name,
-    configuration_data_source_name=None,
-):
+Dump = collections.namedtuple(
+    'Dump',
+    ('hook_name', 'data_source_name', 'hostname', 'port'),
+    defaults=('localhost', None),
+)
+
+
+def dumps_match(first, second):
     '''
-    Find the first data source with the given hook name and data source name in the configuration
-    dict and the given archive data source names dict (from hook name to data source names contained
-    in a particular backup archive). If UNSPECIFIED_HOOK is given as the hook name, search all data
-    source hooks for the named data source. If a configuration data source name is given, use that
-    instead of the data source name to lookup the data source in the given hooks configuration.
-
-    Return the found data source as a tuple of (found hook name, data source configuration dict) or
-    (None, None) if not found.
+    Compare two Dump instances for equality while supporting a field value of UNSPECIFIED, which
+    indicates that the field should match any value.
     '''
-    if not configuration_data_source_name:
-        configuration_data_source_name = data_source_name
+    for field_name in first._fields:
+        first_value = getattr(first, field_name)
+        second_value = getattr(second, field_name)
 
-    if hook_name == UNSPECIFIED_HOOK:
-        hooks_to_search = {
-            hook_name: value
-            for (hook_name, value) in config.items()
-            if hook_name.split('_databases')[0]
-            in borgmatic.hooks.dispatch.get_submodule_names(borgmatic.hooks.data_source)
-        }
+        if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
+            continue
+
+        if first_value != second_value:
+            return False
+
+    return True
+
+
+def render_dump_metadata(dump):
+    '''
+    Given a Dump instance, make a display string describing it for use in log messages.
+    '''
+    name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
+    hostname = dump.hostname or 'localhost'
+    port = None if dump.port is UNSPECIFIED else dump.port
+
+    if port:
+        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
     else:
-        try:
-            hooks_to_search = {hook_name: config[hook_name]}
-        except KeyError:
-            return (None, None)
+        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
 
-    return next(
-        (
-            (name, hook_data_source)
-            for (name, hook) in hooks_to_search.items()
-            for hook_data_source in hook
-            if hook_data_source['name'] == configuration_data_source_name
-            and data_source_name in archive_data_source_names.get(name, [])
-        ),
-        (None, None),
+    if dump.hook_name not in (None, UNSPECIFIED):
+        return f'{metadata} ({dump.hook_name})'
+
+    return metadata
+
+
+def get_configured_data_source(config, restore_dump):
+    '''
+    Search in the given configuration dict for dumps corresponding to the given dump to restore. If
+    there are multiple matches, error.
+
+    Return the found data source as a data source configuration dict or None if not found.
+    '''
+    try:
+        hooks_to_search = {restore_dump.hook_name: config[restore_dump.hook_name]}
+    except KeyError:
+        return None
+
+    matching_dumps = tuple(
+        hook_data_source
+        for (hook_name, hook) in hooks_to_search.items()
+        for hook_data_source in hook
+        if dumps_match(
+            Dump(
+                hook_name,
+                hook_data_source.get('name'),
+                hook_data_source.get('hostname', 'localhost'),
+                hook_data_source.get('port'),
+            ),
+            restore_dump,
+        )
     )
 
+    if not matching_dumps:
+        return None
+
+    if len(matching_dumps) > 1:
+        raise ValueError(
+            f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
+        )
+
+    return matching_dumps[0]
+
 
 def strip_path_prefix_from_extracted_dump_destination(
     destination_path, borgmatic_runtime_directory
@@ -98,7 +135,7 @@ def strip_path_prefix_from_extracted_dump_destination(
         break
 
 
-def restore_single_data_source(
+def restore_single_dump(
     repository,
     config,
     local_borg_version,
@@ -116,8 +153,12 @@ def restore_single_data_source(
     username/password as connection params, and a configured data source configuration dict, restore
     that data source from the archive.
     '''
+    dump_metadata = render_dump_metadata(
+        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
+    )
+
     logger.info(
-        f'{repository.get("label", repository["path"])}: Restoring data source {data_source["name"]}'
+        f'{repository.get("label", repository["path"])}: Restoring data source {dump_metadata}'
     )
 
     dump_patterns = borgmatic.hooks.dispatch.call_hooks(
@@ -127,7 +168,7 @@ def restore_single_data_source(
         borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
         borgmatic_runtime_directory,
         data_source['name'],
-    )[hook_name.split('_databases')[0]]
+    )[hook_name.split('_databases', 1)[0]]
 
     destination_path = (
         tempfile.mkdtemp(dir=borgmatic_runtime_directory)
@@ -180,7 +221,7 @@ def restore_single_data_source(
     )
 
 
-def collect_archive_data_source_names(
+def collect_dumps_from_archive(
     repository,
     archive,
     config,
@@ -192,17 +233,17 @@ def collect_archive_data_source_names(
 ):
     '''
     Given a local or remote repository path, a resolved archive name, a configuration dict, the
-    local Borg version, global_arguments an argparse.Namespace, local and remote Borg paths, and the
-    borgmatic runtime directory, query the archive for the names of data sources it contains as
-    dumps and return them as a dict from hook name to a sequence of data source names.
+    local Borg version, global arguments an argparse.Namespace, local and remote Borg paths, and the
+    borgmatic runtime directory, query the archive for the names of data sources dumps it contains
+    and return them as a set of Dump instances.
     '''
     borgmatic_source_directory = str(
         pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
     )
 
     # Probe for the data source dumps in multiple locations, as the default location has moved to
-    # the borgmatic runtime directory (which get stored as just "/borgmatic" with Borg 1.4+). But we
-    # still want to support reading dumps from previously created archives as well.
+    # the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
+    # we still want to support reading dumps from previously created archives as well.
     dump_paths = borgmatic.borg.list.capture_archive_listing(
         repository,
         archive,
@@ -224,110 +265,145 @@ def collect_archive_data_source_names(
         remote_path=remote_path,
     )
 
-    # Determine the data source names corresponding to the dumps found in the archive and
-    # add them to restore_names.
-    archive_data_source_names = {}
+    # Parse the paths of dumps found in the archive to get their respective dump metadata.
+    dumps_from_archive = set()
 
     for dump_path in dump_paths:
         if not dump_path:
             continue
 
+        # Probe to find the base directory that's at the start of the dump path.
         for base_directory in (
             'borgmatic',
             borgmatic_runtime_directory,
             borgmatic_source_directory,
         ):
             try:
-                (hook_name, _, data_source_name) = dump_path.split(base_directory + os.path.sep, 1)[
-                    1
-                ].split(os.path.sep)[0:3]
+                (hook_name, host_and_port, data_source_name) = dump_path.split(
+                    base_directory + os.path.sep, 1
+                )[1].split(os.path.sep)[0:3]
             except (ValueError, IndexError):
-                pass
-            else:
-                if data_source_name not in archive_data_source_names.get(hook_name, []):
-                    archive_data_source_names.setdefault(hook_name, []).extend([data_source_name])
-                    break
+                continue
+
+            parts = host_and_port.split(':', 1)
+
+            if len(parts) == 1:
+                parts += (None,)
+
+            (hostname, port) = parts
+
+            try:
+                port = int(port)
+            except (ValueError, TypeError):
+                port = None
+
+            dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
+
+            # We've successfully parsed the dump path, so need to probe any further.
+            break
         else:
             logger.warning(
                 f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
             )
 
-    return archive_data_source_names
+    return dumps_from_archive
 
 
-def find_data_sources_to_restore(requested_data_source_names, archive_data_source_names):
+def get_dumps_to_restore(restore_arguments, dumps_from_archive):
     '''
-    Given a sequence of requested data source names to restore and a dict of hook name to the names
-    of data sources found in an archive, return an expanded sequence of data source names to
-    restore, replacing "all" with actual data source names as appropriate.
+    Given restore arguments as an argparse.Namespace instance indicating which dumps to restore and
+    a set of Dump instances representing the dumps found in an archive, return a set of specific
+    Dump instances from the archive to restore. As part of this, replace any Dump having a data
+    source name of "all" with multiple named Dump instances as appropriate.
 
-    Raise ValueError if any of the requested data source names cannot be found in the archive.
+    Raise ValueError if any of the requested data source names cannot be found in the archive or if
+    there are multiple archive dump matches for a given requested dump.
     '''
-    # A map from data source hook name to the data source names to restore for that hook.
-    restore_names = (
-        {UNSPECIFIED_HOOK: requested_data_source_names}
-        if requested_data_source_names
-        else {UNSPECIFIED_HOOK: ['all']}
+    requested_dumps = (
+        {
+            Dump(
+                hook_name=(
+                    (
+                        restore_arguments.hook
+                        if restore_arguments.hook.endswith('_databases')
+                        else f'{restore_arguments.hook}_databases'
+                    )
+                    if restore_arguments.hook
+                    else UNSPECIFIED
+                ),
+                data_source_name=name,
+                hostname=restore_arguments.original_hostname or 'localhost',
+                port=restore_arguments.original_port,
+            )
+            for name in restore_arguments.data_sources
+        }
+        if restore_arguments.data_sources
+        else {
+            Dump(
+                hook_name=UNSPECIFIED,
+                data_source_name='all',
+                hostname=UNSPECIFIED,
+                port=UNSPECIFIED,
+            )
+        }
     )
+    missing_dumps = set()
+    dumps_to_restore = set()
 
-    # If "all" is in restore_names, then replace it with the names of dumps found within the
-    # archive.
-    if 'all' in restore_names[UNSPECIFIED_HOOK]:
-        restore_names[UNSPECIFIED_HOOK].remove('all')
+    # If there's a requested "all" dump, add every dump from the archive to the dumps to restore.
+    if any(dump for dump in requested_dumps if dump.data_source_name == 'all'):
+        dumps_to_restore.update(dumps_from_archive)
 
-        for hook_name, data_source_names in archive_data_source_names.items():
-            restore_names.setdefault(hook_name, []).extend(data_source_names)
+    # If any archive dump matches a requested dump, add the archive dump to the dumps to restore.
+    for requested_dump in requested_dumps:
+        if requested_dump.data_source_name == 'all':
+            continue
 
-            # If a data source is to be restored as part of "all", then remove it from restore names
-            # so it doesn't get restored twice.
-            for data_source_name in data_source_names:
-                if data_source_name in restore_names[UNSPECIFIED_HOOK]:
-                    restore_names[UNSPECIFIED_HOOK].remove(data_source_name)
-
-    if not restore_names[UNSPECIFIED_HOOK]:
-        restore_names.pop(UNSPECIFIED_HOOK)
-
-    combined_restore_names = set(
-        name for data_source_names in restore_names.values() for name in data_source_names
-    )
-    combined_archive_data_source_names = set(
-        name
-        for data_source_names in archive_data_source_names.values()
-        for name in data_source_names
-    )
-
-    missing_names = sorted(set(combined_restore_names) - combined_archive_data_source_names)
-    if missing_names:
-        joined_names = ', '.join(f'"{name}"' for name in missing_names)
-        raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from archive"
+        matching_dumps = tuple(
+            archive_dump
+            for archive_dump in dumps_from_archive
+            if dumps_match(requested_dump, archive_dump)
         )
 
-    return restore_names
+        if len(matching_dumps) == 0:
+            missing_dumps.add(requested_dump)
+        elif len(matching_dumps) == 1:
+            dumps_to_restore.add(matching_dumps[0])
+        else:
+            raise ValueError(
+                f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
+            )
+
+    if missing_dumps:
+        rendered_dumps = ', '.join(
+            f'{render_dump_metadata(dump)}' for dump in sorted(missing_dumps)
+        )
+
+        raise ValueError(
+            f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
+        )
+
+    return dumps_to_restore
 
 
-def ensure_data_sources_found(restore_names, remaining_restore_names, found_names):
+def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
     '''
-    Given a dict from hook name to data source names to restore, a dict from hook name to remaining
-    data source names to restore, and a sequence of found (actually restored) data source names,
-    raise ValueError if requested data source to restore were missing from the archive and/or
+    Given a set of requested dumps to restore and a set of dumps actually restored, raise ValueError
+    if any requested dumps to restore weren't restored, indicating that they were missing from the
     configuration.
     '''
-    combined_restore_names = set(
-        name
-        for data_source_names in tuple(restore_names.values())
-        + tuple(remaining_restore_names.values())
-        for name in data_source_names
-    )
-
-    if not combined_restore_names and not found_names:
+    if not dumps_actually_restored:
         raise ValueError('No data source dumps were found to restore')
 
-    missing_names = sorted(set(combined_restore_names) - set(found_names))
-    if missing_names:
-        joined_names = ', '.join(f'"{name}"' for name in missing_names)
+    missing_dumps = sorted(
+        dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
+    )
+
+    if missing_dumps:
+        rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
+
         raise ValueError(
-            f"Cannot restore data source{'s' if len(missing_names) > 1 else ''} {joined_names} missing from borgmatic's configuration"
+            f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
         )
 
 
@@ -344,7 +420,8 @@ def run_restore(
     Run the "restore" action for the given repository, but only if the repository matches the
     requested repository in restore arguments.
 
-    Raise ValueError if a configured data source could not be found to restore.
+    Raise ValueError if a configured data source could not be found to restore or there's no
+    matching dump in the archive.
     '''
     if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
         repository, restore_arguments.repository
@@ -375,7 +452,7 @@ def run_restore(
             local_path,
             remote_path,
         )
-        archive_data_source_names = collect_archive_data_source_names(
+        dumps_from_archive = collect_dumps_from_archive(
             repository['path'],
             archive_name,
             config,
@@ -385,11 +462,9 @@ def run_restore(
             remote_path,
             borgmatic_runtime_directory,
         )
-        restore_names = find_data_sources_to_restore(
-            restore_arguments.data_sources, archive_data_source_names
-        )
-        found_names = set()
-        remaining_restore_names = {}
+        dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
+
+        dumps_actually_restored = set()
         connection_params = {
             'hostname': restore_arguments.hostname,
             'port': restore_arguments.port,
@@ -398,61 +473,42 @@ def run_restore(
             'restore_path': restore_arguments.restore_path,
         }
 
-        for hook_name, data_source_names in restore_names.items():
-            for data_source_name in data_source_names:
-                found_hook_name, found_data_source = get_configured_data_source(
-                    config, archive_data_source_names, hook_name, data_source_name
-                )
+        # Restore each dump.
+        for restore_dump in dumps_to_restore:
+            found_data_source = get_configured_data_source(
+                config,
+                restore_dump,
+            )
 
-                if not found_data_source:
-                    remaining_restore_names.setdefault(found_hook_name or hook_name, []).append(
-                        data_source_name
-                    )
-                    continue
-
-                found_names.add(data_source_name)
-                restore_single_data_source(
-                    repository,
+            # For a dump that wasn't found via an exact match in the configuration, try to fallback
+            # to an "all" data source.
+            if not found_data_source:
+                found_data_source = get_configured_data_source(
                     config,
-                    local_borg_version,
-                    global_arguments,
-                    local_path,
-                    remote_path,
-                    archive_name,
-                    found_hook_name or hook_name,
-                    dict(found_data_source, **{'schemas': restore_arguments.schemas}),
-                    connection_params,
-                    borgmatic_runtime_directory,
-                )
-
-        # For any data sources that weren't found via exact matches in the configuration, try to
-        # fallback to "all" entries.
-        for hook_name, data_source_names in remaining_restore_names.items():
-            for data_source_name in data_source_names:
-                found_hook_name, found_data_source = get_configured_data_source(
-                    config, archive_data_source_names, hook_name, data_source_name, 'all'
+                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
                 )
 
                 if not found_data_source:
                     continue
 
-                found_names.add(data_source_name)
-                data_source = copy.copy(found_data_source)
-                data_source['name'] = data_source_name
+                found_data_source = dict(found_data_source)
+                found_data_source['name'] = restore_dump.data_source_name
 
-                restore_single_data_source(
-                    repository,
-                    config,
-                    local_borg_version,
-                    global_arguments,
-                    local_path,
-                    remote_path,
-                    archive_name,
-                    found_hook_name or hook_name,
-                    dict(data_source, **{'schemas': restore_arguments.schemas}),
-                    connection_params,
-                    borgmatic_runtime_directory,
-                )
+            dumps_actually_restored.add(restore_dump)
+
+            restore_single_dump(
+                repository,
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path,
+                remote_path,
+                archive_name,
+                restore_dump.hook_name,
+                dict(found_data_source, **{'schemas': restore_arguments.schemas}),
+                connection_params,
+                borgmatic_runtime_directory,
+            )
 
         borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
             'remove_data_source_dumps',
@@ -463,4 +519,4 @@ def run_restore(
             global_arguments.dry_run,
         )
 
-    ensure_data_sources_found(restore_names, remaining_restore_names, found_names)
+    ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored)

borgmatic/borg/create.py

@@ -160,14 +160,24 @@ def any_parent_directories(path, candidate_parents):
 
 
 def collect_special_file_paths(
-    create_command, config, local_path, working_directory, borg_environment, skip_directories
+    create_command,
+    config,
+    local_path,
+    working_directory,
+    borg_environment,
+    borgmatic_runtime_directory,
 ):
     '''
     Given a Borg create command as a tuple, a configuration dict, a local Borg path, a working
-    directory, a dict of environment variables to pass to Borg, and a sequence of parent directories
-    to skip, collect the paths for any special files (character devices, block devices, and named
-    pipes / FIFOs) that Borg would encounter during a create. These are all paths that could cause
-    Borg to hang if its --read-special flag is used.
+    directory, a dict of environment variables to pass to Borg, and the borgmatic runtime directory,
+    collect the paths for any special files (character devices, block devices, and named pipes /
+    FIFOs) that Borg would encounter during a create. These are all paths that could cause Borg to
+    hang if its --read-special flag is used.
+
+    Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
+    its own special files there for database dumps. And if the borgmatic runtime directory is
+    configured to be excluded from the files Borg backs up, error, because this means Borg won't be
+    able to consume any database dumps and therefore borgmatic will hang.
     '''
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
     # files including any named pipe we've created.
@@ -186,12 +196,19 @@ def collect_special_file_paths(
         for path_line in paths_output.split('\n')
         if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
     )
+    skip_paths = {}
 
-    return tuple(
-        path
-        for path in paths
-        if special_file(path) and not any_parent_directories(path, skip_directories)
-    )
+    if os.path.exists(borgmatic_runtime_directory):
+        skip_paths = {
+            path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
+        }
+
+        if not skip_paths:
+            raise ValueError(
+                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes. Please remove it from excludes or change the runtime directory.'
+            )
+
+    return tuple(path for path in paths if special_file(path) if path not in skip_paths)
 
 
 def check_all_source_directories_exist(source_directories):
@@ -335,9 +352,7 @@ def make_base_create_command(
             local_path,
             working_directory,
             borg_environment,
-            skip_directories=(
-                [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else []
-            ),
+            borgmatic_runtime_directory=borgmatic_runtime_directory,
         )
 
         if special_file_paths:

borgmatic/commands/arguments.py

@@ -1153,7 +1153,7 @@ def make_parsers():
         metavar='NAME',
         dest='data_sources',
         action='append',
-        help="Name of data source (e.g. database) to restore from archive, must be defined in borgmatic's configuration, can specify flag multiple times, defaults to all data sources in the archive",
+        help="Name of data source (e.g. database) to restore from the archive, must be defined in borgmatic's configuration, can specify the flag multiple times, defaults to all data sources in the archive",
     )
     restore_group.add_argument(
         '--schema',
@@ -1182,6 +1182,19 @@ def make_parsers():
         '--restore-path',
         help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--original-hostname',
+        help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
+    restore_group.add_argument(
+        '--original-port',
+        type=int,
+        help="The port where the dump to restore came from (if that port is in borgmatic's configuration), only necessary if you need to disambiguate dumps",
+    )
+    restore_group.add_argument(
+        '--hook',
+        help='The name of the data source hook for the dump to restore, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )

borgmatic/commands/borgmatic.py

@@ -8,8 +8,6 @@ import time
 from queue import Queue
 from subprocess import CalledProcessError
 
-import colorama
-
 import borgmatic.actions.borg
 import borgmatic.actions.break_lock
 import borgmatic.actions.change_passphrase
@@ -915,7 +913,7 @@ def main(extra_summary_logs=[]):  # pragma: no cover
         getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
     )
     color_enabled = should_do_markup(global_arguments.no_color or any_json_flags, configs)
-    colorama.init(autoreset=color_enabled, strip=not color_enabled)
+
     try:
         configure_logging(
             verbosity_to_log_level(global_arguments.verbosity),

borgmatic/config/schema.yaml

@@ -68,9 +68,7 @@ properties:
         type: boolean
         description: |
             Stay in same file system; do not cross mount points beyond the given
-            source directories. Defaults to false. But when a database hook is
-            used, the setting here is ignored and one_file_system is considered
-            true.
+            source directories. Defaults to false.
         example: true
     numeric_ids:
         type: boolean
@@ -963,8 +961,8 @@ properties:
                         dump all databases on the host. (Also set the "format"
                         to dump each database to a separate file instead of one
                         combined file.) Note that using this database hook
-                        implicitly enables both read_special and one_file_system
-                        (see above) to support dump and restore streaming.
+                        implicitly enables read_special (see above) to support
+                        dump and restore streaming.
                     example: users
                 hostname:
                     type: string
@@ -1145,9 +1143,8 @@ properties:
                     description: |
                         Database name (required if using this hook). Or "all" to
                         dump all databases on the host. Note that using this
-                        database hook implicitly enables both read_special and
-                        one_file_system (see above) to support dump and restore
-                        streaming.
+                        database hook implicitly enables read_special (see
+                        above) to support dump and restore streaming.
                     example: users
                 hostname:
                     type: string
@@ -1272,9 +1269,8 @@ properties:
                     description: |
                         Database name (required if using this hook). Or "all" to
                         dump all databases on the host. Note that using this
-                        database hook implicitly enables both read_special and
-                        one_file_system (see above) to support dump and restore
-                        streaming.
+                        database hook implicitly enables read_special (see
+                        above) to support dump and restore streaming.
                     example: users
                 hostname:
                     type: string
@@ -1407,9 +1403,9 @@ properties:
                     description: |
                         Path to the SQLite database file to dump. If relative,
                         it is relative to the current working directory. Note
-                        that using this database hook implicitly enables both
-                        read_special and one_file_system (see above) to support
-                        dump and restore streaming.
+                        that using this database hook implicitly enables
+                        read_special (see above) to support dump and restore
+                        streaming.
                     example: /var/lib/sqlite/users.db
                 restore_path:
                     type: string
@@ -1429,9 +1425,8 @@ properties:
                     description: |
                         Database name (required if using this hook). Or "all" to
                         dump all databases on the host. Note that using this
-                        database hook implicitly enables both read_special and
-                        one_file_system (see above) to support dump and restore
-                        streaming.
+                        database hook implicitly enables read_special (see
+                        above) to support dump and restore streaming.
                     example: users
                 hostname:
                     type: string

borgmatic/hooks/data_source/bootstrap.py

@@ -34,7 +34,7 @@ def dump_data_sources(
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.
     '''
-    if hook_config.get('store_config_files') is False:
+    if hook_config and hook_config.get('store_config_files') is False:
         return []
 
     borgmatic_manifest_path = os.path.join(

borgmatic/hooks/data_source/btrfs.py

@@ -1,5 +1,6 @@
 import collections
 import glob
+import json
 import logging
 import os
 import shutil
@@ -26,13 +27,21 @@ def get_filesystem_mount_points(findmnt_command):
     findmnt_output = borgmatic.execute.execute_command_and_capture_output(
         tuple(findmnt_command.split(' '))
         + (
-            '-n',  # No headings.
             '-t',  # Filesystem type.
             'btrfs',
+            '--json',
+            '--list',  # Request a flat list instead of a nested subvolume hierarchy.
         )
     )
 
-    return tuple(line.rstrip().split(' ')[0] for line in findmnt_output.splitlines())
+    try:
+        return tuple(
+            filesystem['target'] for filesystem in json.loads(findmnt_output)['filesystems']
+        )
+    except json.JSONDecodeError as error:
+        raise ValueError(f'Invalid {findmnt_command} JSON output: {error}')
+    except KeyError as error:
+        raise ValueError(f'Invalid {findmnt_command} output: Missing key "{error}"')
 
 
 def get_subvolumes_for_filesystem(btrfs_command, filesystem_mount_point):
@@ -257,8 +266,12 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_
     '''
     Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic runtime
     directory, and whether this is a dry run, delete any Btrfs snapshots created by borgmatic. Use
-    the log prefix in any log entries. If this is a dry run, then don't actually remove anything.
+    the log prefix in any log entries. If this is a dry run or Btrfs isn't configured in borgmatic's
+    configuration, then don't actually remove anything.
     '''
+    if hook_config is None:
+        return
+
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     btrfs_command = hook_config.get('btrfs_command', 'btrfs')

borgmatic/hooks/data_source/dump.py

@@ -16,17 +16,19 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam
     return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
 
 
-def make_data_source_dump_filename(dump_path, name, hostname=None):
+def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
     '''
-    Based on the given dump directory path, data source name, and hostname, return a filename to use
-    for the data source dump. The hostname defaults to localhost.
+    Based on the given dump directory path, data source name, hostname, and port, return a filename
+    to use for the data source dump. The hostname defaults to localhost.
 
     Raise ValueError if the data source name is invalid.
     '''
     if os.path.sep in name:
         raise ValueError(f'Invalid data source name {name}')
 
-    return os.path.join(dump_path, hostname or 'localhost', name)
+    return os.path.join(
+        dump_path, (hostname or 'localhost') + ('' if port is None else f':{port}'), name
+    )
 
 
 def create_parent_directory_for_dump(dump_path):

borgmatic/hooks/data_source/lvm.py

@@ -89,6 +89,8 @@ def snapshot_logical_volume(
             '--snapshot',
             ('--extents' if '%' in snapshot_size else '--size'),
             snapshot_size,
+            '--permission',
+            'r',  # Read-only.
             '--name',
             snapshot_name,
             logical_volume_device,
@@ -286,9 +288,12 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_
     '''
     Given an LVM configuration dict, a configuration dict, a log prefix, the borgmatic runtime
     directory, and whether this is a dry run, unmount and delete any LVM snapshots created by
-    borgmatic. Use the log prefix in any log entries. If this is a dry run, then don't actually
-    remove anything.
+    borgmatic. Use the log prefix in any log entries. If this is a dry run or LVM isn't configured
+    in borgmatic's configuration, then don't actually remove anything.
     '''
+    if hook_config is None:
+        return
+
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     # Unmount snapshots.

borgmatic/hooks/data_source/mariadb.py

@@ -73,7 +73,10 @@ def execute_dump_command(
     '''
     database_name = database['name']
     dump_filename = dump.make_data_source_dump_filename(
-        dump_path, database['name'], database.get('hostname')
+        dump_path,
+        database['name'],
+        database.get('hostname'),
+        database.get('port'),
     )
 
     if os.path.exists(dump_filename):

borgmatic/hooks/data_source/mongodb.py

@@ -51,7 +51,10 @@ def dump_data_sources(
     for database in databases:
         name = database['name']
         dump_filename = dump.make_data_source_dump_filename(
-            make_dump_path(borgmatic_runtime_directory), name, database.get('hostname')
+            make_dump_path(borgmatic_runtime_directory),
+            name,
+            database.get('hostname'),
+            database.get('port'),
         )
         dump_format = database.get('format', 'archive')
 

borgmatic/hooks/data_source/mysql.py

@@ -73,7 +73,10 @@ def execute_dump_command(
     '''
     database_name = database['name']
     dump_filename = dump.make_data_source_dump_filename(
-        dump_path, database['name'], database.get('hostname')
+        dump_path,
+        database['name'],
+        database.get('hostname'),
+        database.get('port'),
     )
 
     if os.path.exists(dump_filename):

borgmatic/hooks/data_source/postgresql.py

@@ -151,7 +151,10 @@ def dump_data_sources(
                 for part in shlex.split(database.get('pg_dump_command') or default_dump_command)
             )
             dump_filename = dump.make_data_source_dump_filename(
-                dump_path, database_name, database.get('hostname')
+                dump_path,
+                database_name,
+                database.get('hostname'),
+                database.get('port'),
             )
             if os.path.exists(dump_filename):
                 logger.warning(

borgmatic/hooks/data_source/zfs.py

@@ -283,9 +283,12 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_
     '''
     Given a ZFS configuration dict, a configuration dict, a log prefix, the borgmatic runtime
     directory, and whether this is a dry run, unmount and destroy any ZFS snapshots created by
-    borgmatic. Use the log prefix in any log entries. If this is a dry run, then don't actually
-    remove anything.
+    borgmatic. Use the log prefix in any log entries. If this is a dry run or ZFS isn't configured
+    in borgmatic's configuration, then don't actually remove anything.
     '''
+    if hook_config is None:
+        return
+
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     # Unmount snapshots.

borgmatic/hooks/dispatch.py

@@ -32,7 +32,11 @@ def call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs):
     Raise AttributeError if the function name is not found in the module.
     Raise anything else that the called function raises.
     '''
-    hook_config = config.get(hook_name) or config.get(f'{hook_name}_databases') or {}
+    if hook_name in config or f'{hook_name}_databases' in config:
+        hook_config = config.get(hook_name) or config.get(f'{hook_name}_databases') or {}
+    else:
+        hook_config = None
+
     module_name = hook_name.split('_databases')[0]
 
     # Probe for a data source or monitoring hook module corresponding to the hook name.

borgmatic/logger.py

@@ -1,10 +1,9 @@
+import enum
 import logging
 import logging.handlers
 import os
 import sys
 
-import colorama
-
 
 def to_bool(arg):
     '''
@@ -33,7 +32,7 @@ def interactive_console():
 def should_do_markup(no_color, configs):
     '''
     Given the value of the command-line no-color argument, and a dict of configuration filename to
-    corresponding parsed configuration, determine if we should enable colorama marking up.
+    corresponding parsed configuration, determine if we should enable color marking up.
     '''
     if no_color:
         return False
@@ -93,30 +92,50 @@ class Console_no_color_formatter(logging.Formatter):
         return record.msg
 
 
+class Color(enum.Enum):
+    RESET = 0
+    RED = 31
+    GREEN = 32
+    YELLOW = 33
+    MAGENTA = 35
+    CYAN = 36
+
+
 class Console_color_formatter(logging.Formatter):
     def format(self, record):
         add_custom_log_levels()
 
-        color = {
-            logging.CRITICAL: colorama.Fore.RED,
-            logging.ERROR: colorama.Fore.RED,
-            logging.WARN: colorama.Fore.YELLOW,
-            logging.ANSWER: colorama.Fore.MAGENTA,
-            logging.INFO: colorama.Fore.GREEN,
-            logging.DEBUG: colorama.Fore.CYAN,
-        }.get(record.levelno)
+        color = (
+            {
+                logging.CRITICAL: Color.RED,
+                logging.ERROR: Color.RED,
+                logging.WARN: Color.YELLOW,
+                logging.ANSWER: Color.MAGENTA,
+                logging.INFO: Color.GREEN,
+                logging.DEBUG: Color.CYAN,
+            }
+            .get(record.levelno)
+            .value
+        )
 
         return color_text(color, record.msg)
 
 
+def ansi_escape_code(color):  # pragma: no cover
+    '''
+    Given a color value, produce the corresponding ANSI escape code.
+    '''
+    return f'\x1b[{color}m'
+
+
 def color_text(color, message):
     '''
-    Give colored text.
+    Given a color value and a message, return the message colored with ANSI escape codes.
     '''
     if not color:
         return message
 
-    return f'{color}{message}{colorama.Style.RESET_ALL}'
+    return f'{ansi_escape_code(color)}{message}{ansi_escape_code(Color.RESET.value)}'
 
 
 def add_logging_level(level_name, level_number):

docs/how-to/backup-your-databases.md

@@ -391,7 +391,7 @@ with the repository's path or its label as configured in your borgmatic
 configuration file.
 
 ```bash
-borgmatic restore --repository repo.borg --archive host-2023-...
+borgmatic restore --repository repo.borg --archive latest
 ```
 
 ### Restore particular databases
@@ -401,7 +401,7 @@ restore one of them, use the `--database` flag to select one or more
 databases. For instance:
 
 ```bash
-borgmatic restore --archive host-2023-... --database users --database orders
+borgmatic restore --archive latest --database users --database orders
 ```
 
 <span class="minilink minilink-addedin">New in version 1.7.6</span> You can
@@ -409,29 +409,99 @@ also restore individual databases even if you dumped them as "all"—as long as
 you dumped them into separate files via use of the "format" option. See above
 for more information.
 
+### Restore databases sharing a name
+
+<span class="minilink minilink-addedin">New in version 1.9.5</span> If you've
+backed up multiple databases that happen to share the same name but different
+hostnames, ports, or hooks, you can include additional flags to disambiguate
+which database you'd like to restore. For instance, let's say you've backed up
+the following configured databases:
+
+```yaml
+postgresql_databases:
+    - name: users
+      hostname: host1.example.org
+    - name: users
+      hostname: host2.example.org
+```
+
+... then you can run the following command to restore only one of them:
+
+```bash
+borgmatic restore --archive latest --database users --original-hostname host1.example.org
+```
+
+This selects a `users` database to restore, but only if it originally came
+from the host `host1.example.org`. This command won't restore `users`
+databases from any other hosts.
+
+Here's another example configuration:
+
+```yaml
+postgresql_databases:
+    - name: users
+      hostname: example.org
+      port: 5433
+    - name: users
+      hostname: example.org
+      port: 5434
+```
+
+And a command to restore just one of the databases:
+
+```bash
+borgmatic restore --archive latest --database users --original-port 5433
+```
+
+That restores a `users` database only if it originally came from port `5433`
+*and* if that port is in borgmatic's configuration, e.g. `port: 5433`.
+
+Finally, check out this configuration:
+
+```yaml
+postgresql_databases:
+    - name: users
+      hostname: example.org
+mariadb_databases:
+    - name: users
+      hostname: example.org
+```
+
+And to select just one of the databases to restore:
+
+```bash
+borgmatic restore --archive latest --database users --hook postgresql
+```
+
+That restores a `users` database only if it was dumped using the
+`postgresql_databases:` data source hook. This command won't restore `users`
+databases that were dumped using other hooks.
+
+Note that these flags don't change the hostname or port to which the database
+is actually restored. For that, see below about restoring to an alternate
+host.
+
 
 ### Restore all databases
 
 To restore all databases:
 
 ```bash
-borgmatic restore --archive host-2023-... --database all
+borgmatic restore --archive latest --database all
 ```
 
 Or omit the `--database` flag entirely:
 
 
 ```bash
-borgmatic restore --archive host-2023-...
+borgmatic restore --archive latest
 ```
 
-Prior to borgmatic version 1.7.6, this restores a combined "all" database
-dump from the archive.
-
 <span class="minilink minilink-addedin">New in version 1.7.6</span> Restoring
 "all" databases restores each database found in the selected archive. That
 includes any combined dump file named "all" and any other individual database
-dumps found in the archive.
+dumps found in the archive. Prior to borgmatic version 1.7.6, restoring "all"
+only restored a combined "all" database dump from the archive.
 
 
 ### Restore particular schemas
@@ -466,21 +536,35 @@ postgresql_databases:
         restore_password: trustsome1
 ```
 
+### Manual restoration
 
-### Limitations
+If you prefer to restore a database without the help of borgmatic, first
+[extract](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) an
+archive containing a database dump.
+
+borgmatic extracts the dump file into the `borgmatic/` directory within the
+extraction destination path. For example, if you're extracting to `/tmp`, then
+the dump will be in `/tmp/borgmatic/`.
+
+<span class="minilink minilink-addedin">Prior to version 1.9.0</span> borgmatic
+extracts the dump file into the *`username`*`/.borgmatic/` directory within the
+extraction destination path, where *`username`* is the user that created the
+backup. For example, if you created the backup with the `root` user and you're
+extracting to `/tmp`, then the dump will be in `/tmp/root/.borgmatic`.
+
+After extraction, you can manually restore the dump file using native database
+commands like `pg_restore`, `mysql`, `mongorestore`, `sqlite`, or similar.
+
+Also see the documentation on [listing database
+dumps](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#listing-database-dumps).
+
+
+## Limitations
 
 There are a few important limitations with borgmatic's current database
-restoration feature that you should know about:
+hooks that you should know about:
 
-1. You must restore as the same Unix user that created the archive containing
-the database dump. That's because the user's home directory path is encoded
-into the path of the database dump within the archive.
-2. As mentioned above, borgmatic can only restore a database that's defined in
-borgmatic's own configuration file. So include your configuration file in
-backups to avoid getting caught without a way to restore a database.
-3. borgmatic does not currently support backing up or restoring multiple
-databases that share the exact same name on different hosts.
-4. When database hooks are enabled, borgmatic instructs Borg to consume
+1. When database hooks are enabled, borgmatic instructs Borg to consume
 special files (via `--read-special`) to support database dump
 streaming—regardless of the value of your `read_special` configuration option.
 And because this can cause Borg to hang, borgmatic also automatically excludes
@@ -495,7 +579,10 @@ exclude them. <span class="minilink minilink-addedin">Prior to version
 1.7.3</span>Special files were not auto-excluded, and you were responsible for
 excluding them yourself. Common directories to exclude are `/dev` and `/run`,
 but that may not be exhaustive.
-5. <span class="minilink minilink-addedin">Prior to version 1.9.0</span>
+2. <span class="minilink minilink-addedin">Prior to version 1.9.5</span>
+borgmatic did not support backing up or restoring multiple databases that
+shared the exact same name on different hosts or with different ports.
+3. <span class="minilink minilink-addedin">Prior to version 1.9.0</span>
 Database hooks also implicitly enabled the `one_file_system` option, which
 meant Borg wouldn't cross filesystem boundaries when looking for files to
 backup. When borgmatic was running in a container, this often required a
@@ -503,25 +590,16 @@ work-around to explicitly add each mounted backup volume to
 `source_directories` instead of relying on Borg to include them implicitly via
 a parent directory. But as of borgmatic 1.9.0, `one_file_system` is no longer
 auto-enabled and such work-arounds aren't necessary.
-
-
-### Manual restoration
-
-If you prefer to restore a database without the help of borgmatic, first
-[extract](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) an
-archive containing a database dump.
-
-borgmatic extracts the dump file into the *`username`*`/.borgmatic/` directory
-within the extraction destination path, where *`username`* is the user that
-created the backup. For example, if you created the backup with the `root`
-user and you're extracting to `/tmp`, then the dump will be in
-`/tmp/root/.borgmatic`.
-
-After extraction, you can manually restore the dump file using native database
-commands like `pg_restore`, `mysql`, `mongorestore`, `sqlite`, or similar.
-
-Also see the documentation on [listing database
-dumps](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/#listing-database-dumps).
+4. <span class="minilink minilink-addedin">Prior to version 1.9.0</span> You
+must restore as the same Unix user that created the archive containing the
+database dump. That's because the user's home directory path is encoded into
+the path of the database dump within the archive.
+5. <span class="minilink minilink-addedin">Prior to version 1.7.15</span> As
+mentioned above, borgmatic can only restore a database that's defined in
+borgmatic's own configuration file. So include your configuration files in
+backups to avoid getting caught without a way to restore a database. But
+starting from version 1.7.15, borgmatic includes your configuration files
+automatically.
 
 
 ## Preparation and cleanup hooks

docs/how-to/snapshot-your-filesystems.md

@@ -63,7 +63,7 @@ like particular datasets to be backed up only for particular configuration
 files, use the `source_directories` option instead of the user property.
 
 During a backup, borgmatic automatically snapshots these discovered datasets
-(non-recursively), temporary mounts the snapshots within its [runtime
+(non-recursively), temporarily mounts the snapshots within its [runtime
 directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory),
 and includes the snapshotted files in the paths sent to Borg. borgmatic is also
 responsible for cleaning up (destroying) these snapshots after a backup
@@ -215,12 +215,12 @@ too small (and LVM isn't configured to grow snapshots automatically), then the
 snapshots will fail to allocate enough space, resulting in a broken backup.
 
 If not specified, the `snapshot_size` option defaults to `10%ORIGIN`, which
-means 10% of the size of logical volume being snapshotted. See the [`lvcreate
---size` and `--extents`
+means 10% of the size of the logical volume being snapshotted. See the
+[`lvcreate --size` and `--extents`
 documentation](https://www.man7.org/linux/man-pages/man8/lvcreate.8.html) for
 more information about possible values here. (Under the hood, borgmatic uses
-`lvcreate --extents` if the `snapshot_size` is a percentage value, and
-`lvcreate --size` otherwise.)
+`lvcreate --extents` if the `snapshot_size` is a percentage value, and `lvcreate
+--size` otherwise.)
 
 
 #### Logical volume discovery
@@ -228,12 +228,11 @@ more information about possible values here. (Under the hood, borgmatic uses
 For any logical volume you'd like backed up, add its mount point to
 borgmatic's `source_directories` option.
 
-During a backup, borgmatic automatically snapshots these discovered logical
-volumes (non-recursively), temporary mounts the snapshots within its [runtime
-directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory),
-and includes the snapshotted files in the paths sent to Borg. borgmatic is
-also responsible for cleaning up (deleting) these snapshots after a backup
-completes.
+During a backup, borgmatic automatically snapshots these discovered logical volumes, temporarily
+mounts the snapshots within its [runtime
+directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory), and
+includes the snapshotted files in the paths sent to Borg. borgmatic is also responsible for cleaning
+up (deleting) these snapshots after a backup completes.
 
 borgmatic is smart enough to look at the parent (and grandparent, etc.)
 directories of each of your `source_directories` to discover any logical

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "borgmatic"
-version = "1.9.4"
+version = "1.9.5"
 authors = [
   { name="Dan Helfman", email="witten@torsion.org" },
 ]
@@ -17,11 +17,10 @@ classifiers=[
     "Topic :: System :: Archiving :: Backup",
 ]
 dependencies = [
-  "colorama>=0.4.1,<0.5",
-  "jsonschema",
-  "packaging",
-  "requests",
-  "ruamel.yaml>0.15.0",
+    "jsonschema",
+    "packaging",
+    "requests",
+    "ruamel.yaml>0.15.0",
 ]
 
 [project.scripts]

test_requirements.txt

@@ -6,7 +6,6 @@ certifi==2024.7.4
 chardet==5.2.0
 click==8.1.7
 codespell==2.2.6
-colorama==0.4.6
 coverage==7.5.1
 flake8==7.0.0
 flake8-quotes==3.4.0

tests/end-to-end/commands/fake_findmnt.py

@@ -4,29 +4,38 @@ import sys
 
 def parse_arguments(*unparsed_arguments):
     parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument('-n', dest='headings', action='store_false', default=True)
     parser.add_argument('-t', dest='type')
+    parser.add_argument('--json', action='store_true')
+    parser.add_argument('--list', action='store_true')
 
     return parser.parse_args(unparsed_arguments)
 
 
-BUILTIN_FILESYSTEM_MOUNT_LINES = (
-    '/mnt/subvolume /dev/loop1 btrfs rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/',
-)
+BUILTIN_FILESYSTEM_MOUNT_OUTPUT = '''{
+       "filesystems": [
+          {
+             "target": "/mnt/subvolume",
+             "source": "/dev/loop0",
+             "fstype": "btrfs",
+             "options": "rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/"
+          }
+       ]
+    }
+    '''
 
 
-def print_filesystem_mounts(arguments):
-    for line in BUILTIN_FILESYSTEM_MOUNT_LINES:
-        print(line)
+def print_filesystem_mounts():
+    print(BUILTIN_FILESYSTEM_MOUNT_OUTPUT)
 
 
 def main():
     arguments = parse_arguments(*sys.argv[1:])
 
-    assert not arguments.headings
     assert arguments.type == 'btrfs'
+    assert arguments.json
+    assert arguments.list
 
-    print_filesystem_mounts(arguments)
+    print_filesystem_mounts()
 
 
 if __name__ == '__main__':

tests/end-to-end/commands/fake_lvcreate.py

@@ -9,6 +9,7 @@ def parse_arguments(*unparsed_arguments):
     parser.add_argument('--snapshot', action='store_true', required=True)
     parser.add_argument('--extents')
     parser.add_argument('--size')
+    parser.add_argument('--permission', required=True)
     parser.add_argument('--name', dest='snapshot_name', required=True)
     parser.add_argument('logical_volume_device')
 

tests/end-to-end/test_database.py

@@ -16,6 +16,8 @@ def write_configuration(
     repository_path,
     user_runtime_directory,
     postgresql_dump_format='custom',
+    postgresql_all_dump_format=None,
+    mariadb_mysql_all_dump_format=None,
     mongodb_dump_format='archive',
 ):
     '''
@@ -23,6 +25,13 @@ def write_configuration(
     for testing. This includes injecting the given repository path, borgmatic source directory for
     storing database dumps, dump format (for PostgreSQL), and encryption passphrase.
     '''
+    postgresql_all_format_option = (
+        f'format: {postgresql_all_dump_format}' if postgresql_all_dump_format else ''
+    )
+    mariadb_mysql_dump_format_option = (
+        f'format: {mariadb_mysql_all_dump_format}' if mariadb_mysql_all_dump_format else ''
+    )
+
     config_yaml = f'''
 source_directories:
     - {source_directory}
@@ -39,11 +48,7 @@ postgresql_databases:
       password: test
       format: {postgresql_dump_format}
     - name: all
-      hostname: postgresql
-      username: postgres
-      password: test
-    - name: all
-      format: custom
+      {postgresql_all_format_option}
       hostname: postgresql
       username: postgres
       password: test
@@ -53,11 +58,7 @@ mariadb_databases:
       username: root
       password: test
     - name: all
-      hostname: mariadb
-      username: root
-      password: test
-    - name: all
-      format: sql
+      {mariadb_mysql_dump_format_option}
       hostname: mariadb
       username: root
       password: test
@@ -67,11 +68,7 @@ mysql_databases:
       username: root
       password: test
     - name: all
-      hostname: not-actually-mysql
-      username: root
-      password: test
-    - name: all
-      format: sql
+      {mariadb_mysql_dump_format_option}
       hostname: not-actually-mysql
       username: root
       password: test
@@ -97,12 +94,21 @@ sqlite_databases:
     return ruamel.yaml.YAML(typ='safe').load(config_yaml)
 
 
+@pytest.mark.parametrize(
+    'postgresql_all_dump_format,mariadb_mysql_all_dump_format',
+    (
+        (None, None),
+        ('custom', 'sql'),
+    ),
+)
 def write_custom_restore_configuration(
     source_directory,
     config_path,
     repository_path,
     user_runtime_directory,
     postgresql_dump_format='custom',
+    postgresql_all_dump_format=None,
+    mariadb_mysql_all_dump_format=None,
     mongodb_dump_format='archive',
 ):
     '''

tests/unit/borg/test_create.py

@@ -275,7 +275,8 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_
         'Processing files ...\n- /foo\n+ /bar\n- /baz'
     )
     flexmock(module).should_receive('special_file').and_return(True)
-    flexmock(module).should_receive('any_parent_directories').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module).should_receive('any_parent_directories').never()
 
     assert module.collect_special_file_paths(
         ('borg', 'create'),
@@ -283,17 +284,24 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_
         local_path=None,
         working_directory=None,
         borg_environment=None,
-        skip_directories=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/bar', '/baz')
 
 
-def test_collect_special_file_paths_excludes_requested_directories():
+def test_collect_special_file_paths_skips_borgmatic_runtime_directory():
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
-        '+ /foo\n- /bar\n- /baz'
+        '+ /foo\n- /run/borgmatic/bar\n- /baz'
     )
     flexmock(module).should_receive('special_file').and_return(True)
-    flexmock(module).should_receive('any_parent_directories').and_return(False).and_return(
-        True
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module).should_receive('any_parent_directories').with_args(
+        '/foo', ('/run/borgmatic',)
+    ).and_return(False)
+    flexmock(module).should_receive('any_parent_directories').with_args(
+        '/run/borgmatic/bar', ('/run/borgmatic',)
+    ).and_return(True)
+    flexmock(module).should_receive('any_parent_directories').with_args(
+        '/baz', ('/run/borgmatic',)
     ).and_return(False)
 
     assert module.collect_special_file_paths(
@@ -302,10 +310,29 @@ def test_collect_special_file_paths_excludes_requested_directories():
         local_path=None,
         working_directory=None,
         borg_environment=None,
-        skip_directories=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/baz')
 
 
+def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_from_paths_output_errors():
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(
+        '+ /foo\n- /bar\n- /baz'
+    )
+    flexmock(module).should_receive('special_file').and_return(True)
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module).should_receive('any_parent_directories').and_return(False)
+
+    with pytest.raises(ValueError):
+        module.collect_special_file_paths(
+            ('borg', 'create'),
+            config={},
+            local_path=None,
+            working_directory=None,
+            borg_environment=None,
+            borgmatic_runtime_directory='/run/borgmatic',
+        )
+
+
 def test_collect_special_file_paths_excludes_non_special_files():
     flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         '+ /foo\n+ /bar\n+ /baz'
@@ -313,7 +340,8 @@ def test_collect_special_file_paths_excludes_non_special_files():
     flexmock(module).should_receive('special_file').and_return(True).and_return(False).and_return(
         True
     )
-    flexmock(module).should_receive('any_parent_directories').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module).should_receive('any_parent_directories').never()
 
     assert module.collect_special_file_paths(
         ('borg', 'create'),
@@ -321,7 +349,7 @@ def test_collect_special_file_paths_excludes_non_special_files():
         local_path=None,
         working_directory=None,
         borg_environment=None,
-        skip_directories=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/foo', '/baz')
 
 
@@ -335,7 +363,8 @@ def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command():
         borg_exit_codes=None,
     ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once()
     flexmock(module).should_receive('special_file').and_return(True)
-    flexmock(module).should_receive('any_parent_directories').and_return(False)
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module).should_receive('any_parent_directories').never()
 
     module.collect_special_file_paths(
         ('borg', 'create', '--exclude-nodump'),
@@ -343,7 +372,7 @@ def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command():
         local_path='borg',
         working_directory=None,
         borg_environment=None,
-        skip_directories=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     )
 
 

tests/unit/hooks/data_source/test_bootstrap.py

@@ -22,7 +22,7 @@ def test_dump_data_sources_creates_manifest_file():
     ).once()
 
     module.dump_data_sources(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         config_paths=('test.yaml',),
@@ -53,7 +53,7 @@ def test_dump_data_sources_with_dry_run_does_not_create_manifest_file():
     flexmock(module.json).should_receive('dump').never()
 
     module.dump_data_sources(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         config_paths=('test.yaml',),
@@ -74,7 +74,7 @@ def test_remove_data_source_dumps_deletes_manifest_and_parent_directory():
     flexmock(module.os).should_receive('rmdir').with_args('/run/borgmatic/bootstrap').once()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',
@@ -91,7 +91,7 @@ def test_remove_data_source_dumps_with_dry_run_bails():
     flexmock(module.os).should_receive('rmdir').never()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',
@@ -110,7 +110,7 @@ def test_remove_data_source_dumps_swallows_manifest_file_not_found_error():
     flexmock(module.os).should_receive('rmdir').with_args('/run/borgmatic/bootstrap').once()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',
@@ -131,7 +131,7 @@ def test_remove_data_source_dumps_swallows_manifest_parent_directory_not_found_e
     ).once()
 
     module.remove_data_source_dumps(
-        hook_config={},
+        hook_config=None,
         config={},
         log_prefix='test',
         borgmatic_runtime_directory='/run/borgmatic',

tests/unit/hooks/data_source/test_btrfs.py

@@ -1,3 +1,4 @@
+import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks.data_source import btrfs as module
@@ -7,13 +8,46 @@ def test_get_filesystem_mount_points_parses_findmnt_output():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
     ).and_return(
-        '/mnt0   /dev/loop0 btrfs  rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/\n'
-        '/mnt1   /dev/loop1 btrfs  rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/\n'
+        '''{
+           "filesystems": [
+              {
+                 "target": "/mnt0",
+                 "source": "/dev/loop0",
+                 "fstype": "btrfs",
+                 "options": "rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/"
+              },
+              {
+                 "target": "/mnt1",
+                 "source": "/dev/loop0",
+                 "fstype": "btrfs",
+                 "options": "rw,relatime,ssd,space_cache=v2,subvolid=5,subvol=/"
+              }
+           ]
+        }
+        '''
     )
 
     assert module.get_filesystem_mount_points('findmnt') == ('/mnt0', '/mnt1')
 
 
+def test_get_filesystem_mount_points_with_invalid_findmnt_json_errors():
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).and_return('{')
+
+    with pytest.raises(ValueError):
+        module.get_filesystem_mount_points('findmnt')
+
+
+def test_get_filesystem_mount_points_with_findmnt_json_missing_filesystems_errors():
+    flexmock(module.borgmatic.execute).should_receive(
+        'execute_command_and_capture_output'
+    ).and_return('{"wtf": "something is wrong here"}')
+
+    with pytest.raises(ValueError):
+        module.get_filesystem_mount_points('findmnt')
+
+
 def test_get_subvolumes_for_filesystem_parses_subvolume_list_output():
     flexmock(module.borgmatic.execute).should_receive(
         'execute_command_and_capture_output'
@@ -451,6 +485,24 @@ def test_remove_data_source_dumps_deletes_snapshots():
     )
 
 
+def test_remove_data_source_dumps_without_hook_configuration_bails():
+    flexmock(module).should_receive('get_subvolumes').never()
+    flexmock(module).should_receive('make_snapshot_path').never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'replace_temporary_subdirectory_with_glob'
+    ).never()
+    flexmock(module).should_receive('delete_snapshot').never()
+    flexmock(module.shutil).should_receive('rmtree').never()
+
+    module.remove_data_source_dumps(
+        hook_config=None,
+        config={'source_directories': '/mnt/subvolume'},
+        log_prefix='test',
+        borgmatic_runtime_directory='/run/borgmatic',
+        dry_run=False,
+    )
+
+
 def test_remove_data_source_dumps_with_get_subvolumes_file_not_found_error_bails():
     config = {'btrfs': {}}
     flexmock(module).should_receive('get_subvolumes').and_raise(FileNotFoundError)

tests/unit/hooks/data_source/test_dump.py

@@ -15,6 +15,13 @@ def test_make_data_source_dump_filename_uses_name_and_hostname():
     )
 
 
+def test_make_data_source_dump_filename_uses_name_and_hostname_and_port():
+    assert (
+        module.make_data_source_dump_filename('databases', 'test', 'hostname', 1234)
+        == 'databases/hostname:1234/test'
+    )
+
+
 def test_make_data_source_dump_filename_without_hostname_defaults_to_localhost():
     assert module.make_data_source_dump_filename('databases', 'test') == 'databases/localhost/test'
 

tests/unit/hooks/data_source/test_lvm.py

@@ -99,6 +99,8 @@ def test_snapshot_logical_volume_with_percentage_snapshot_name_uses_lvcreate_ext
             '--snapshot',
             '--extents',
             '10%ORIGIN',
+            '--permission',
+            'r',
             '--name',
             'snap',
             '/dev/snap',
@@ -116,6 +118,8 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
             '--snapshot',
             '--size',
             '10TB',
+            '--permission',
+            'r',
             '--name',
             'snap',
             '/dev/snap',
@@ -669,6 +673,23 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots():
     )
 
 
+def test_remove_data_source_dumps_bails_for_missing_lvm_configuration():
+    flexmock(module).should_receive('get_logical_volumes').never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'replace_temporary_subdirectory_with_glob'
+    ).never()
+    flexmock(module).should_receive('unmount_snapshot').never()
+    flexmock(module).should_receive('remove_snapshot').never()
+
+    module.remove_data_source_dumps(
+        hook_config=None,
+        config={'source_directories': '/mnt/lvolume'},
+        log_prefix='test',
+        borgmatic_runtime_directory='/run/borgmatic',
+        dry_run=False,
+    )
+
+
 def test_remove_data_source_dumps_bails_for_missing_lsblk_command():
     config = {'lvm': {}}
     flexmock(module).should_receive('get_logical_volumes').and_raise(FileNotFoundError)

tests/unit/hooks/data_source/test_zfs.py

@@ -321,6 +321,21 @@ def test_remove_data_source_dumps_use_custom_commands():
     )
 
 
+def test_remove_data_source_dumps_bails_for_missing_hook_configuration():
+    flexmock(module).should_receive('get_all_dataset_mount_points').never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'replace_temporary_subdirectory_with_glob'
+    ).never()
+
+    module.remove_data_source_dumps(
+        hook_config=None,
+        config={'source_directories': '/mnt/dataset'},
+        log_prefix='test',
+        borgmatic_runtime_directory='/run/borgmatic',
+        dry_run=False,
+    )
+
+
 def test_remove_data_source_dumps_bails_for_missing_zfs_command():
     flexmock(module).should_receive('get_all_dataset_mount_points').and_raise(FileNotFoundError)
     flexmock(module.borgmatic.config.paths).should_receive(

tests/unit/hooks/test_dispatch.py

@@ -95,7 +95,7 @@ def test_call_hook_without_hook_config_invokes_module_function_with_arguments_an
         'borgmatic.hooks.monitoring.super_hook'
     ).and_return(test_module)
     flexmock(test_module).should_receive('hook_function').with_args(
-        {}, config, 'prefix', 55, value=66
+        None, config, 'prefix', 55, value=66
     ).and_return(expected_return_value).once()
 
     return_value = module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66)

tests/unit/test_logger.py

@@ -187,10 +187,14 @@ def test_console_color_formatter_format_includes_log_message():
 
 
 def test_color_text_does_not_raise():
-    module.color_text(module.colorama.Fore.RED, 'hi')
+    flexmock(module).should_receive('ansi_escape_code').and_return('blah')
+
+    module.color_text(module.Color.RED, 'hi')
 
 
 def test_color_text_without_color_does_not_raise():
+    flexmock(module).should_receive('ansi_escape_code').and_return('blah')
+
     module.color_text(None, 'hi')