Read and write data source dump metadata files within an archive.

borgmatic/actions/restore.py

@@ -252,6 +252,59 @@ def collect_dumps_from_archive(
     borgmatic runtime directory, query the archive for the names of data sources dumps it contains
     and return them as a set of Dump instances.
     '''
+    dumps_from_archive = set()
+
+    # There is (at most) one dump metadata file per data source hook. Load each.
+    for dump_metadata_path in borgmatic.borg.list.capture_archive_listing(
+        repository,
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        list_paths=[
+            'sh:'
+            + borgmatic.hooks.data_source.dump.make_data_source_dump_path(
+                base_directory,
+                '*_databases/dumps.json',
+            )
+            # Probe for dump metadata files in multiple locations, as the default location is
+            # "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the
+            # borgmatic runtime directory for older versions of Borg.
+            for base_directory in (
+                'borgmatic',
+                borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+            )
+        ],
+        local_path=local_path,
+        remote_path=remote_path,
+    ):
+        dumps_from_archive.update(
+            set(
+                borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata(
+                    borgmatic.borg.extract.extract_archive(
+                        global_arguments.dry_run,
+                        repository,
+                        archive,
+                        [dump_metadata_path],
+                        config,
+                        local_borg_version,
+                        global_arguments,
+                        local_path=local_path,
+                        remote_path=remote_path,
+                        extract_to_stdout=True,
+                    )
+                    .stdout.read()
+                    .decode()
+                )
+            )
+        )
+
+    # If we've successfully loaded any dumps metadata, we're done.
+    if dumps_from_archive:
+        return dumps_from_archive
+
+    # No dumps metadata files were found, so for backwards compatibility, fall back to parsing the
+    # paths of dumps found in the archive to get their respective dump metadata.
     borgmatic_source_directory = str(
         pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)),
     )
@@ -281,9 +334,6 @@ def collect_dumps_from_archive(
         remote_path=remote_path,
     )
 
-    # Parse the paths of dumps found in the archive to get their respective dump metadata.
-    dumps_from_archive = set()
-
     for dump_path in dump_paths:
         if not dump_path:
             continue

borgmatic/hooks/data_source/dump.py

@@ -1,8 +1,11 @@
 import fnmatch
+import json
 import logging
 import os
 import shutil
 
+import borgmatic.actions.restore
+
 logger = logging.getLogger(__name__)
 
 IS_A_HOOK = False
@@ -33,6 +36,30 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
     )
 
 
+def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata):
+    '''
+    Given the borgmatic runtime directory, a data source hook name, and a sequence of
+    borgmatic.actions.restore.Dump instances of dump metadata, write a metadata file describing all
+    of those dumps. This metadata is being dumped so that it's available upon restore, e.g. to
+    support the user selecting which data source(s) should be restored.
+    '''
+    # TODO: Handle file errors?
+    with open(
+        os.path.join(borgmatic_runtime_directory, hook_name, 'dumps.json'), 'w'
+    ) as metadata_file:
+        json.dump([dump._asdict() for dump in dumps_metadata], metadata_file)
+
+
+def parse_data_source_dumps_metadata(dumps_json):
+    '''
+    Given a dumps metadata JSON string as extracted from an archive, parse it into a tuple of
+    borgmatic.actions.restore.Dump instances and return them.
+    '''
+    # TODO: Deal with JSON parse errors.
+    # TODO: Deal with wrong JSON data for the Dump() constructor.
+    return tuple(borgmatic.actions.restore.Dump(**dump) for dump in json.loads(dumps_json))
+
+
 def create_parent_directory_for_dump(dump_path):
     '''
     Create a directory to contain the given dump path.

borgmatic/hooks/data_source/postgresql.py

@@ -150,12 +150,13 @@ def dump_data_sources(
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
     processes = []
+    dumps_metadata = []
 
     logger.info(f'Dumping PostgreSQL databases{dry_run_label}')
+    dump_path = make_dump_path(borgmatic_runtime_directory)
 
     for database in databases:
         environment = make_environment(database, config)
-        dump_path = make_dump_path(borgmatic_runtime_directory)
         dump_database_names = database_names_to_dump(database, config, environment, dry_run)
 
         if not dump_database_names:
@@ -165,6 +166,14 @@ def dump_data_sources(
             raise ValueError('Cannot find any PostgreSQL databases to dump.')
 
         for database_name in dump_database_names:
+            dumps_metadata.append(
+                borgmatic.actions.restore.Dump(
+                    'postgresql_databases',
+                    database_name,
+                    database.get('hostname'),
+                    database.get('port'),
+                )
+            )
             dump_format = database.get('format', None if database_name == 'all' else 'custom')
             compression = database.get('compression')
             default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump'
@@ -178,6 +187,7 @@ def dump_data_sources(
                 database.get('hostname'),
                 database.get('port'),
             )
+
             if os.path.exists(dump_filename):
                 logger.warning(
                     f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}',
@@ -246,6 +256,11 @@ def dump_data_sources(
                     ),
                 )
 
+        if not dry_run:
+            dump.write_data_source_dumps_metadata(
+                borgmatic_runtime_directory, 'postgresql_databases', dumps_metadata
+            )
+
     if not dry_run:
         patterns.append(
             borgmatic.borg.pattern.Pattern(