Read and write data source dump metadata files within an archive (#1136).

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1136

NEWS

@@ -11,6 +11,7 @@
    with orphaned files that need recovery.
  * #1133: Fix the "spot" check to include borgmatic configuration files that were backed up to
    support the "bootstrap" action.
+ * #1136: For all database hooks, record metadata about the dumps contained within an archive.
  * #1139: Set "borgmatic" as the user agent when connecting to monitoring services.
  * When running tests, use Ruff for faster and more comprehensive code linting and formatting,
    replacing Flake8, Black, isort, etc.

borgmatic/actions/restore.py

@@ -252,6 +252,66 @@ def collect_dumps_from_archive(
     borgmatic runtime directory, query the archive for the names of data sources dumps it contains
     and return them as a set of Dump instances.
     '''
+    dumps_from_archive = set()
+
+    # There is (at most) one dump metadata file per data source hook. Load each.
+    for dumps_metadata_path in borgmatic.borg.list.capture_archive_listing(
+        repository,
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        list_paths=[
+            'sh:'
+            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
+                base_directory,
+                '*_databases/dumps.json',
+            )
+            # Probe for dump metadata files in multiple locations, as the default location is
+            # "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the
+            # borgmatic runtime directory for older versions of Borg.
+            for base_directory in (
+                'borgmatic',
+                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+            )
+        ],
+        local_path=local_path,
+        remote_path=remote_path,
+    ):
+        if not dumps_metadata_path:
+            continue
+
+        dumps_from_archive.update(
+            set(
+                borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata(
+                    borgmatic.borg.extract.extract_archive(
+                        global_arguments.dry_run,
+                        repository,
+                        archive,
+                        [dumps_metadata_path],
+                        config,
+                        local_borg_version,
+                        global_arguments,
+                        local_path=local_path,
+                        remote_path=remote_path,
+                        extract_to_stdout=True,
+                    )
+                    .stdout.read()
+                    .decode(),
+                    dumps_metadata_path,
+                )
+            )
+        )
+
+    # If we've successfully loaded any dumps metadata, we're done.
+    if dumps_from_archive:
+        logger.debug('Collecting database dumps from archive data source dumps metadata files')
+
+        return dumps_from_archive
+
+    # No dumps metadata files were found, so for backwards compatibility, fall back to parsing the
+    # paths of dumps found in the archive to get their respective dump metadata.
+    logger.debug('Collecting database dumps from archive data source dump paths (fallback)')
     borgmatic_source_directory = str(
         pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)),
     )
@@ -281,9 +341,6 @@ def collect_dumps_from_archive(
         remote_path=remote_path,
     )
 
-    # Parse the paths of dumps found in the archive to get their respective dump metadata.
-    dumps_from_archive = set()
-
     for dump_path in dump_paths:
         if not dump_path:
             continue

borgmatic/hooks/data_source/dump.py

@@ -1,8 +1,11 @@
 import fnmatch
+import json
 import logging
 import os
 import shutil
 
+import borgmatic.actions.restore
+
 logger = logging.getLogger(__name__)
 
 IS_A_HOOK = False
@@ -33,6 +36,48 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
     )
 
 
+def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata):
+    '''
+    Given the borgmatic runtime directory, a data source hook name, and a sequence of
+    borgmatic.actions.restore.Dump instances of dump metadata, write a metadata file describing all
+    of those dumps. This metadata is being dumped so that it's available upon restore, e.g. to
+    support the user selecting which data source(s) should be restored.
+
+    Raise ValueError if writing to the file results in an operating system error.
+    '''
+    dumps_metadata_path = os.path.join(borgmatic_runtime_directory, hook_name, 'dumps.json')
+
+    try:
+        with open(dumps_metadata_path, 'w', encoding='utf-8') as metadata_file:
+            json.dump(
+                {
+                    'dumps': [dump._asdict() for dump in dumps_metadata],
+                },
+                metadata_file,
+                sort_keys=True,
+            )
+    except OSError as error:
+        raise ValueError(f'Error writing to dumps metadata at {dumps_metadata_path}: {error}')
+
+
+def parse_data_source_dumps_metadata(dumps_json, dumps_metadata_path):
+    '''
+    Given a dumps metadata JSON string as extracted from an archive and its path within the archive,
+    parse it into a tuple of borgmatic.actions.restore.Dump instances and return them.
+
+    Raise ValueError if parsing the JSON results in a JSON decode error or the data does not have
+    the expected keys.
+    '''
+    try:
+        return tuple(
+            borgmatic.actions.restore.Dump(**dump) for dump in json.loads(dumps_json)['dumps']
+        )
+    except (json.JSONDecodeError, TypeError, KeyError) as error:
+        raise ValueError(
+            f'Cannot read archive data source dumps metadata at {dumps_metadata_path} due to invalid JSON: {error}',
+        )
+
+
 def create_parent_directory_for_dump(dump_path):
     '''
     Create a directory to contain the given dump path.

borgmatic/hooks/data_source/mariadb.py

@@ -257,11 +257,12 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping MariaDB databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         username = borgmatic.hooks.credential.parse.resolve_credential(
             database.get('username'),
             config,
@@ -294,9 +295,17 @@ def dump_data_sources(
             raise ValueError('Cannot find any MariaDB databases to dump.')
 
         if database['name'] == 'all' and database.get('format'):
-            for dump_name in dump_database_names:
+            for database_name in dump_database_names:
+                dumps_metadata.append(
+                    borgmatic.actions.restore.Dump(
+                        'mariadb_databases',
+                        database_name,
+                        database.get('hostname', 'localhost'),
+                        database.get('port'),
+                    )
+                )
                 renamed_database = copy.copy(database)
-                renamed_database['name'] = dump_name
+                renamed_database['name'] = database_name
                 processes.append(
                     execute_dump_command(
                         renamed_database,
@@ -304,13 +313,21 @@ def dump_data_sources(
                         username,
                         password,
                         dump_path,
-                        (dump_name,),
+                        (database_name,),
                         environment,
                         dry_run,
                         dry_run_label,
                     ),
                 )
         else:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'mariadb_databases',
+                    database['name'],
+                    database.get('hostname', 'localhost'),
+                    database.get('port'),
+                )
+            )
             processes.append(
                 execute_dump_command(
                     database,
@@ -326,6 +343,9 @@ def dump_data_sources(
             )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mariadb_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mariadb_databases'),

borgmatic/hooks/data_source/mongodb.py

@@ -53,9 +53,19 @@ def dump_data_sources(
     logger.info(f'Dumping MongoDB databases{dry_run_label}')
 
     processes = []
+    dumps_metadata = []
 
     for database in databases:
         name = database['name']
+        dumps_metadata.append(
+            borgmatic.actions.restore.Dump(
+                'mongodb_databases',
+                name,
+                database.get('hostname', 'localhost'),
+                database.get('port'),
+            )
+        )
+
         dump_filename = dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
@@ -82,6 +92,9 @@ def dump_data_sources(
             )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mongodb_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mongodb_databases'),

borgmatic/hooks/data_source/mysql.py

@@ -188,11 +188,12 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping MySQL databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         username = borgmatic.hooks.credential.parse.resolve_credential(
             database.get('username'),
             config,
@@ -225,9 +226,17 @@ def dump_data_sources(
             raise ValueError('Cannot find any MySQL databases to dump.')
 
         if database['name'] == 'all' and database.get('format'):
-            for dump_name in dump_database_names:
+            for database_name in dump_database_names:
+                dumps_metadata.append(
+                    borgmatic.actions.restore.Dump(
+                        'mysql_databases',
+                        database_name,
+                        database.get('hostname', 'localhost'),
+                        database.get('port'),
+                    )
+                )
                 renamed_database = copy.copy(database)
-                renamed_database['name'] = dump_name
+                renamed_database['name'] = database_name
                 processes.append(
                     execute_dump_command(
                         renamed_database,
@@ -235,13 +244,21 @@ def dump_data_sources(
                         username,
                         password,
                         dump_path,
-                        (dump_name,),
+                        (database_name,),
                         environment,
                         dry_run,
                         dry_run_label,
                     ),
                 )
         else:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'mysql_databases',
+                    database['name'],
+                    database.get('hostname', 'localhost'),
+                    database.get('port'),
+                )
+            )
             processes.append(
                 execute_dump_command(
                     database,
@@ -257,6 +274,9 @@ def dump_data_sources(
             )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'mysql_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'mysql_databases'),

borgmatic/hooks/data_source/postgresql.py

@@ -150,12 +150,13 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping PostgreSQL databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
         environment = make_environment(database, config)
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
@@ -165,6 +166,14 @@ def dump_data_sources(
             raise ValueError('Cannot find any PostgreSQL databases to dump.')
 
         for database_name in dump_database_names:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'postgresql_databases',
+                    database_name,
+                    database.get('hostname', 'localhost'),
+                    database.get('port'),
+                )
+            )
             dump_format = database.get('format', None if database_name == 'all' else 'custom')
             compression = database.get('compression')
             default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump'
@@ -178,6 +187,7 @@ def dump_data_sources(
                 database.get('hostname'),
                 database.get('port'),
             )
+
             if os.path.exists(dump_filename):
                 logger.warning(
                     f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}',
@@ -247,6 +257,9 @@ def dump_data_sources(
                 )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'postgresql_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'postgresql_databases'),

borgmatic/hooks/data_source/sqlite.py

@@ -49,11 +49,18 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping SQLite databases{dry_run_label}')
 
     for database in databases:
         database_path = database['path']
+        dumps_metadata.append(
+            borgmatic.actions.restore.Dump(
+                'sqlite_databases',
+                database['name'],
+            )
+        )
 
         if database['name'] == 'all':
             logger.warning('The "all" database name has no meaning for SQLite databases')
@@ -95,6 +102,9 @@ def dump_data_sources(
         )
 
     if not dry_run:
+        dump.write_data_source_dumps_metadata(
+            borgmatic_runtime_directory, 'sqlite_databases', dumps_metadata
+        )
         patterns.append(
             borgmatic.borg.pattern.Pattern(
                 os.path.join(borgmatic_runtime_directory, 'sqlite_databases'),

tests/unit/actions/test_restore.py

@@ -462,27 +462,118 @@ def test_restore_single_dump_with_directory_dump_and_dry_run_skips_directory_mov
     )
 
 
-def test_collect_dumps_from_archive_parses_archive_paths():
-    flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/root/.borgmatic')
+def test_collect_dumps_from_archive_with_dumps_metadata_parses_it():
     flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
         'make_data_source_dump_path',
     ).and_return('')
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'make_runtime_directory_glob'
+    ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
         [
-            'borgmatic/postgresql_databases/localhost/foo',
-            'borgmatic/postgresql_databases/host:1234/bar',
-            'borgmatic/mysql_databases/localhost/quux',
+            'borgmatic/postgresql_databases/dumps.json',
+            'borgmatic/mysql_databases/dumps.json',
         ],
     )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        flexmock(stdout=flexmock(read=lambda: b''))
+    )
+    dumps_metadata = [
+        module.Dump('postgresql_databases', 'foo'),
+        module.Dump('postgresql_databases', 'bar', 'host', 1234),
+        module.Dump('mysql_databases', 'quux'),
+    ]
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'parse_data_source_dumps_metadata'
+    ).and_return(dumps_metadata)
+    flexmock(module.borgmatic.config.paths).should_receive('get_borgmatic_source_directory').never()
 
     archive_dumps = module.collect_dumps_from_archive(
         repository={'path': 'repo'},
         archive='archive',
         config={},
         local_borg_version=flexmock(),
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+        local_path=flexmock(),
+        remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+    assert archive_dumps == set(dumps_metadata)
+
+
+def test_collect_dumps_from_archive_with_empty_dumps_metadata_path_falls_back_to_parsing_archive_paths():
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'make_runtime_directory_glob'
+    ).and_return('')
+    flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
+        ['']
+    ).and_return(
+        [
+            'borgmatic/postgresql_databases/localhost/foo',
+            'borgmatic/postgresql_databases/host:1234/bar',
+            'borgmatic/mysql_databases/localhost/quux',
+        ],
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'parse_data_source_dumps_metadata'
+    ).never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/root/.borgmatic')
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path',
+    ).and_return('')
+
+    archive_dumps = module.collect_dumps_from_archive(
+        repository={'path': 'repo'},
+        archive='archive',
+        config={},
+        local_borg_version=flexmock(),
+        global_arguments=flexmock(dry_run=False, log_json=False),
+        local_path=flexmock(),
+        remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+    assert archive_dumps == {
+        module.Dump('postgresql_databases', 'foo'),
+        module.Dump('postgresql_databases', 'bar', 'host', 1234),
+        module.Dump('mysql_databases', 'quux'),
+    }
+
+
+def test_collect_dumps_from_archive_without_dumps_metadata_falls_back_to_parsing_archive_paths():
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'make_runtime_directory_glob'
+    ).and_return('')
+    flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
+        []
+    ).and_return(
+        [
+            'borgmatic/postgresql_databases/localhost/foo',
+            'borgmatic/postgresql_databases/host:1234/bar',
+            'borgmatic/mysql_databases/localhost/quux',
+        ],
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'parse_data_source_dumps_metadata'
+    ).never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/root/.borgmatic')
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path',
+    ).and_return('')
+
+    archive_dumps = module.collect_dumps_from_archive(
+        repository={'path': 'repo'},
+        archive='archive',
+        config={},
+        local_borg_version=flexmock(),
+        global_arguments=flexmock(dry_run=False, log_json=False),
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/borgmatic',
@@ -497,12 +588,11 @@ def test_collect_dumps_from_archive_parses_archive_paths():
 
 def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_directories():
     flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/root/.borgmatic')
-    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
-        'make_data_source_dump_path',
+        'make_runtime_directory_glob'
     ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
+        []
+    ).and_return(
         [
             'borgmatic/postgresql_databases/localhost/foo',
             '.borgmatic/postgresql_databases/localhost/bar',
@@ -510,6 +600,16 @@ def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_dir
             '/var/run/0/borgmatic/mysql_databases/localhost/quux',
         ],
     )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'parse_data_source_dumps_metadata'
+    ).never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/root/.borgmatic')
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path',
+    ).and_return('')
 
     archive_dumps = module.collect_dumps_from_archive(
         repository={'path': 'repo'},
@@ -532,17 +632,26 @@ def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_dir
 
 def test_collect_dumps_from_archive_parses_directory_format_archive_paths():
     flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/root/.borgmatic')
-    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
-        'make_data_source_dump_path',
+        'make_runtime_directory_glob'
     ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
+        []
+    ).and_return(
         [
             'borgmatic/postgresql_databases/localhost/foo/table1',
             'borgmatic/postgresql_databases/localhost/foo/table2',
         ],
     )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'parse_data_source_dumps_metadata'
+    ).never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/root/.borgmatic')
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path',
+    ).and_return('')
 
     archive_dumps = module.collect_dumps_from_archive(
         repository={'path': 'repo'},
@@ -562,12 +671,11 @@ def test_collect_dumps_from_archive_parses_directory_format_archive_paths():
 
 def test_collect_dumps_from_archive_skips_bad_archive_paths_or_bad_path_components():
     flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/root/.borgmatic')
-    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
-        'make_data_source_dump_path',
+        'make_runtime_directory_glob'
     ).and_return('')
     flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
+        []
+    ).and_return(
         [
             'borgmatic/postgresql_databases/localhost/foo',
             'borgmatic/postgresql_databases/localhost:abcd/bar',
@@ -576,6 +684,16 @@ def test_collect_dumps_from_archive_skips_bad_archive_paths_or_bad_path_componen
             '',
         ],
     )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'parse_data_source_dumps_metadata'
+    ).never()
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/root/.borgmatic')
+    flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+        'make_data_source_dump_path',
+    ).and_return('')
 
     archive_dumps = module.collect_dumps_from_archive(
         repository={'path': 'repo'},

tests/unit/hooks/data_source/test_dump.py

@@ -1,3 +1,6 @@
+import io
+import sys
+
 import pytest
 from flexmock import flexmock
 
@@ -31,6 +34,75 @@ def test_make_data_source_dump_filename_with_invalid_name_raises():
         module.make_data_source_dump_filename('databases', 'invalid/name')
 
 
+def test_write_data_source_dumps_metadata_writes_json_to_file():
+    dumps_metadata = [
+        module.borgmatic.actions.restore.Dump('databases', 'foo'),
+        module.borgmatic.actions.restore.Dump('databases', 'bar'),
+    ]
+    dumps_stream = io.StringIO('password')
+    dumps_stream.name = '/run/borgmatic/databases/dumps.json'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args(dumps_stream.name, 'w', encoding='utf-8').and_return(
+        dumps_stream
+    )
+    flexmock(dumps_stream).should_receive('close')  # Prevent close() so getvalue() below works.
+
+    module.write_data_source_dumps_metadata('/run/borgmatic', 'databases', dumps_metadata)
+
+    assert (
+        dumps_stream.getvalue()
+        == '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "port": null}, {"data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "port": null}]}'
+    )
+
+
+def test_write_data_source_dumps_metadata_with_operating_system_error_raises():
+    dumps_metadata = [
+        module.borgmatic.actions.restore.Dump('databases', 'foo'),
+        module.borgmatic.actions.restore.Dump('databases', 'bar'),
+    ]
+    dumps_stream = io.StringIO('password')
+    dumps_stream.name = '/run/borgmatic/databases/dumps.json'
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args(dumps_stream.name, 'w', encoding='utf-8').and_raise(
+        OSError
+    )
+
+    with pytest.raises(ValueError):
+        module.write_data_source_dumps_metadata('/run/borgmatic', 'databases', dumps_metadata)
+
+
+def test_parse_data_source_dumps_metadata_converts_json_to_dump_instances():
+    dumps_json = '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "port": null}, {"data_source_name": "bar", "hook_name": "databases", "hostname": "example.org", "port": 1234}]}'
+
+    assert module.parse_data_source_dumps_metadata(
+        dumps_json, 'borgmatic/databases/dumps.json'
+    ) == (
+        module.borgmatic.actions.restore.Dump('databases', 'foo'),
+        module.borgmatic.actions.restore.Dump('databases', 'bar', 'example.org', 1234),
+    )
+
+
+def test_parse_data_source_dumps_metadata_with_invalid_json_raises():
+    with pytest.raises(ValueError):
+        module.parse_data_source_dumps_metadata('[{', 'borgmatic/databases/dumps.json')
+
+
+def test_parse_data_source_dumps_metadata_with_unknown_keys_raises():
+    dumps_json = (
+        '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "wtf": "is this"}]}'
+    )
+
+    with pytest.raises(ValueError):
+        module.parse_data_source_dumps_metadata(dumps_json, 'borgmatic/databases/dumps.json')
+
+
+def test_parse_data_source_dumps_metadata_with_missing_dumps_key_raises():
+    dumps_json = '{"not": "what we are looking for"}'
+
+    with pytest.raises(ValueError):
+        module.parse_data_source_dumps_metadata(dumps_json, 'borgmatic/databases/dumps.json')
+
+
 def test_create_parent_directory_for_dump_does_not_raise():
     flexmock(module.os).should_receive('makedirs')
 

tests/unit/hooks/data_source/test_mariadb.py

@@ -349,6 +349,15 @@ def test_dump_data_sources_dumps_each_database():
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'),
+        ],
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,
@@ -390,6 +399,13 @@ def test_dump_data_sources_dumps_with_password():
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -434,6 +450,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -465,6 +488,13 @@ def test_dump_data_sources_dumps_all_databases_at_once():
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'all'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -499,6 +529,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'),
+        ],
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,

tests/unit/hooks/data_source/test_mongodb.py

@@ -39,6 +39,15 @@ def test_dump_data_sources_runs_mongodump_for_each_database():
             run_to_completion=False,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'bar'),
+        ],
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,
@@ -60,6 +69,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump():
     ).and_return('databases/localhost/bar')
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
 
     assert (
         module.dump_data_sources(
@@ -75,7 +85,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump():
 
 
 def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
-    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 27018}]
     process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
@@ -89,7 +99,7 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
             '--host',
             'database.example.org',
             '--port',
-            '5433',
+            '27018',
             '--db',
             'foo',
             '--archive',
@@ -99,6 +109,15 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump(
+                'mongodb_databases', 'foo', 'database.example.org', 27018
+            ),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -150,6 +169,13 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -174,6 +200,13 @@ def test_dump_data_sources_runs_mongodump_with_directory_format():
         ('mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'),
         shell=True,
     ).and_return(flexmock()).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -210,6 +243,13 @@ def test_dump_data_sources_runs_mongodump_with_options():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -235,6 +275,13 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'all'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -308,7 +355,7 @@ def test_restore_data_source_dump_runs_mongorestore():
 
 def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port():
     hook_config = [
-        {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None},
+        {'name': 'foo', 'hostname': 'database.example.org', 'port': 27018, 'schemas': None},
     ]
     extract_process = flexmock(stdout=flexmock())
 
@@ -325,7 +372,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port():
             '--host',
             'database.example.org',
             '--port',
-            '5433',
+            '27018',
         ],
         processes=[extract_process],
         output_log_level=logging.DEBUG,
@@ -708,6 +755,13 @@ def test_dump_data_sources_uses_custom_mongodump_command():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,

tests/unit/hooks/data_source/test_mysql.py

@@ -232,6 +232,15 @@ def test_dump_data_sources_dumps_each_database():
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'),
+        ],
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,
@@ -273,6 +282,13 @@ def test_dump_data_sources_dumps_with_password():
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -317,6 +333,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -349,6 +372,14 @@ def test_dump_data_sources_dumps_all_databases_at_once():
         dry_run_label=object,
     ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'all'),
+        ],
+    ).once()
+
     assert module.dump_data_sources(
         databases,
         {},
@@ -382,6 +413,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'),
+        ],
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,

tests/unit/hooks/data_source/test_postgresql.py

@@ -273,6 +273,14 @@ def test_dump_data_sources_runs_pg_dump_for_each_database():
             environment={'PGSSLMODE': 'disable'},
             run_to_completion=False,
         ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -336,6 +344,14 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -366,6 +382,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump():
     ).replace_with(lambda value, config: value)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
 
     assert (
         module.dump_data_sources(
@@ -415,6 +432,15 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump(
+                'postgresql_databases', 'foo', 'database.example.org', 5433
+            ),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -461,6 +487,13 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password():
         environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -507,6 +540,13 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped():
         environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -548,6 +588,13 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format():
         shell=True,
         environment={'PGSSLMODE': 'disable'},
     ).and_return(flexmock()).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -595,6 +642,13 @@ def test_dump_data_sources_runs_pg_dump_with_string_compression():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(processes[0]).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -642,6 +696,13 @@ def test_dump_data_sources_runs_pg_dump_with_integer_compression():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(processes[0]).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -688,6 +749,13 @@ def test_dump_data_sources_runs_pg_dump_with_options():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -720,6 +788,13 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'all'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -764,6 +839,13 @@ def test_dump_data_sources_runs_non_default_pg_dump():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ],
+    ).once()
 
     assert module.dump_data_sources(
         databases,

tests/unit/hooks/data_source/test_sqlite.py

@@ -26,6 +26,13 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists():
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -56,6 +63,14 @@ def test_dump_data_sources_dumps_each_database():
     flexmock(module).should_receive('execute_command').and_return(processes[0]).and_return(
         processes[1],
     )
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database2'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -93,6 +108,13 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped():
         shell=True,
         run_to_completion=False,
     ).and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -135,6 +157,13 @@ def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_ge
         shell=True,
         run_to_completion=False,
     ).and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -163,6 +192,13 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database():
     flexmock(module.os.path).should_receive('exists').and_return(False)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
     flexmock(module).should_receive('execute_command').and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -193,6 +229,13 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases():
     flexmock(module.os.path).should_receive('exists').and_return(False)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
     flexmock(module).should_receive('execute_command').and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'all'),
+        ],
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -217,6 +260,7 @@ def test_dump_data_sources_does_not_dump_if_dry_run():
     flexmock(module.os.path).should_receive('exists').and_return(False)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
 
     assert (
         module.dump_data_sources(