diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 7c9cb99f..36b89884 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -252,6 +252,59 @@ def collect_dumps_from_archive( borgmatic runtime directory, query the archive for the names of data sources dumps it contains and return them as a set of Dump instances. ''' + dumps_from_archive = set() + + # There is (at most) one dump metadata file per data source hook. Load each. + for dump_metadata_path in borgmatic.borg.list.capture_archive_listing( + repository, + archive, + config, + local_borg_version, + global_arguments, + list_paths=[ + 'sh:' + + borgmatic.hooks.data_source.dump.make_data_source_dump_path( + base_directory, + '*_databases/dumps.json', + ) + # Probe for dump metadata files in multiple locations, as the default location is + # "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the + # borgmatic runtime directory for older versions of Borg. + for base_directory in ( + 'borgmatic', + borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory), + ) + ], + local_path=local_path, + remote_path=remote_path, + ): + dumps_from_archive.update( + set( + borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata( + borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + repository, + archive, + [dump_metadata_path], + config, + local_borg_version, + global_arguments, + local_path=local_path, + remote_path=remote_path, + extract_to_stdout=True, + ) + .stdout.read() + .decode() + ) + ) + ) + + # If we've successfully loaded any dumps metadata, we're done. + if dumps_from_archive: + return dumps_from_archive + + # No dumps metadata files were found, so for backwards compatibility, fall back to parsing the + # paths of dumps found in the archive to get their respective dump metadata. borgmatic_source_directory = str( pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)), ) @@ -281,9 +334,6 @@ def collect_dumps_from_archive( remote_path=remote_path, ) - # Parse the paths of dumps found in the archive to get their respective dump metadata. - dumps_from_archive = set() - for dump_path in dump_paths: if not dump_path: continue diff --git a/borgmatic/hooks/data_source/dump.py b/borgmatic/hooks/data_source/dump.py index 205b70c3..e835d0cf 100644 --- a/borgmatic/hooks/data_source/dump.py +++ b/borgmatic/hooks/data_source/dump.py @@ -1,8 +1,11 @@ import fnmatch +import json import logging import os import shutil +import borgmatic.actions.restore + logger = logging.getLogger(__name__) IS_A_HOOK = False @@ -33,6 +36,30 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None): ) +def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata): + ''' + Given the borgmatic runtime directory, a data source hook name, and a sequence of + borgmatic.actions.restore.Dump instances of dump metadata, write a metadata file describing all + of those dumps. This metadata is being dumped so that it's available upon restore, e.g. to + support the user selecting which data source(s) should be restored. + ''' + # TODO: Handle file errors? + with open( + os.path.join(borgmatic_runtime_directory, hook_name, 'dumps.json'), 'w' + ) as metadata_file: + json.dump([dump._asdict() for dump in dumps_metadata], metadata_file) + + +def parse_data_source_dumps_metadata(dumps_json): + ''' + Given a dumps metadata JSON string as extracted from an archive, parse it into a tuple of + borgmatic.actions.restore.Dump instances and return them. + ''' + # TODO: Deal with JSON parse errors. + # TODO: Deal with wrong JSON data for the Dump() constructor. + return tuple(borgmatic.actions.restore.Dump(**dump) for dump in json.loads(dumps_json)) + + def create_parent_directory_for_dump(dump_path): ''' Create a directory to contain the given dump path. diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 123b79a2..5712ecc6 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -150,12 +150,13 @@ def dump_data_sources( ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] + dumps_metadata = [] logger.info(f'Dumping PostgreSQL databases{dry_run_label}') + dump_path = make_dump_path(borgmatic_runtime_directory) for database in databases: environment = make_environment(database, config) - dump_path = make_dump_path(borgmatic_runtime_directory) dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: @@ -165,6 +166,14 @@ def dump_data_sources( raise ValueError('Cannot find any PostgreSQL databases to dump.') for database_name in dump_database_names: + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'postgresql_databases', + database_name, + database.get('hostname'), + database.get('port'), + ) + ) dump_format = database.get('format', None if database_name == 'all' else 'custom') compression = database.get('compression') default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' @@ -178,6 +187,7 @@ def dump_data_sources( database.get('hostname'), database.get('port'), ) + if os.path.exists(dump_filename): logger.warning( f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}', @@ -246,6 +256,11 @@ def dump_data_sources( ), ) + if not dry_run: + dump.write_data_source_dumps_metadata( + borgmatic_runtime_directory, 'postgresql_databases', dumps_metadata + ) + if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern(