diff --git a/NEWS b/NEWS index cb1b43ec..4c0e4f3e 100644 --- a/NEWS +++ b/NEWS @@ -11,6 +11,7 @@ with orphaned files that need recovery. * #1133: Fix the "spot" check to include borgmatic configuration files that were backed up to support the "bootstrap" action. + * #1136: For all database hooks, record metadata about the dumps contained within an archive. * #1139: Set "borgmatic" as the user agent when connecting to monitoring services. * When running tests, use Ruff for faster and more comprehensive code linting and formatting, replacing Flake8, Black, isort, etc. diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 7c9cb99f..fe68d256 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -252,6 +252,66 @@ def collect_dumps_from_archive( borgmatic runtime directory, query the archive for the names of data sources dumps it contains and return them as a set of Dump instances. ''' + dumps_from_archive = set() + + # There is (at most) one dump metadata file per data source hook. Load each. + for dumps_metadata_path in borgmatic.borg.list.capture_archive_listing( + repository, + archive, + config, + local_borg_version, + global_arguments, + list_paths=[ + 'sh:' + + borgmatic.hooks.data_source.dump.make_data_source_dump_path( + base_directory, + '*_databases/dumps.json', + ) + # Probe for dump metadata files in multiple locations, as the default location is + # "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the + # borgmatic runtime directory for older versions of Borg. + for base_directory in ( + 'borgmatic', + borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory), + ) + ], + local_path=local_path, + remote_path=remote_path, + ): + if not dumps_metadata_path: + continue + + dumps_from_archive.update( + set( + borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata( + borgmatic.borg.extract.extract_archive( + global_arguments.dry_run, + repository, + archive, + [dumps_metadata_path], + config, + local_borg_version, + global_arguments, + local_path=local_path, + remote_path=remote_path, + extract_to_stdout=True, + ) + .stdout.read() + .decode(), + dumps_metadata_path, + ) + ) + ) + + # If we've successfully loaded any dumps metadata, we're done. + if dumps_from_archive: + logger.debug('Collecting database dumps from archive data source dumps metadata files') + + return dumps_from_archive + + # No dumps metadata files were found, so for backwards compatibility, fall back to parsing the + # paths of dumps found in the archive to get their respective dump metadata. + logger.debug('Collecting database dumps from archive data source dump paths (fallback)') borgmatic_source_directory = str( pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)), ) @@ -281,9 +341,6 @@ def collect_dumps_from_archive( remote_path=remote_path, ) - # Parse the paths of dumps found in the archive to get their respective dump metadata. - dumps_from_archive = set() - for dump_path in dump_paths: if not dump_path: continue diff --git a/borgmatic/hooks/data_source/dump.py b/borgmatic/hooks/data_source/dump.py index 205b70c3..3b746f7f 100644 --- a/borgmatic/hooks/data_source/dump.py +++ b/borgmatic/hooks/data_source/dump.py @@ -1,8 +1,11 @@ import fnmatch +import json import logging import os import shutil +import borgmatic.actions.restore + logger = logging.getLogger(__name__) IS_A_HOOK = False @@ -33,6 +36,48 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None): ) +def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata): + ''' + Given the borgmatic runtime directory, a data source hook name, and a sequence of + borgmatic.actions.restore.Dump instances of dump metadata, write a metadata file describing all + of those dumps. This metadata is being dumped so that it's available upon restore, e.g. to + support the user selecting which data source(s) should be restored. + + Raise ValueError if writing to the file results in an operating system error. + ''' + dumps_metadata_path = os.path.join(borgmatic_runtime_directory, hook_name, 'dumps.json') + + try: + with open(dumps_metadata_path, 'w', encoding='utf-8') as metadata_file: + json.dump( + { + 'dumps': [dump._asdict() for dump in dumps_metadata], + }, + metadata_file, + sort_keys=True, + ) + except OSError as error: + raise ValueError(f'Error writing to dumps metadata at {dumps_metadata_path}: {error}') + + +def parse_data_source_dumps_metadata(dumps_json, dumps_metadata_path): + ''' + Given a dumps metadata JSON string as extracted from an archive and its path within the archive, + parse it into a tuple of borgmatic.actions.restore.Dump instances and return them. + + Raise ValueError if parsing the JSON results in a JSON decode error or the data does not have + the expected keys. + ''' + try: + return tuple( + borgmatic.actions.restore.Dump(**dump) for dump in json.loads(dumps_json)['dumps'] + ) + except (json.JSONDecodeError, TypeError, KeyError) as error: + raise ValueError( + f'Cannot read archive data source dumps metadata at {dumps_metadata_path} due to invalid JSON: {error}', + ) + + def create_parent_directory_for_dump(dump_path): ''' Create a directory to contain the given dump path. diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index b7770a42..50d66ea6 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -257,11 +257,12 @@ def dump_data_sources( ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] + dumps_metadata = [] logger.info(f'Dumping MariaDB databases{dry_run_label}') + dump_path = make_dump_path(borgmatic_runtime_directory) for database in databases: - dump_path = make_dump_path(borgmatic_runtime_directory) username = borgmatic.hooks.credential.parse.resolve_credential( database.get('username'), config, @@ -294,9 +295,17 @@ def dump_data_sources( raise ValueError('Cannot find any MariaDB databases to dump.') if database['name'] == 'all' and database.get('format'): - for dump_name in dump_database_names: + for database_name in dump_database_names: + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'mariadb_databases', + database_name, + database.get('hostname', 'localhost'), + database.get('port'), + ) + ) renamed_database = copy.copy(database) - renamed_database['name'] = dump_name + renamed_database['name'] = database_name processes.append( execute_dump_command( renamed_database, @@ -304,13 +313,21 @@ def dump_data_sources( username, password, dump_path, - (dump_name,), + (database_name,), environment, dry_run, dry_run_label, ), ) else: + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'mariadb_databases', + database['name'], + database.get('hostname', 'localhost'), + database.get('port'), + ) + ) processes.append( execute_dump_command( database, @@ -326,6 +343,9 @@ def dump_data_sources( ) if not dry_run: + dump.write_data_source_dumps_metadata( + borgmatic_runtime_directory, 'mariadb_databases', dumps_metadata + ) patterns.append( borgmatic.borg.pattern.Pattern( os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index b10dc46c..3a249aaa 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -53,9 +53,19 @@ def dump_data_sources( logger.info(f'Dumping MongoDB databases{dry_run_label}') processes = [] + dumps_metadata = [] for database in databases: name = database['name'] + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'mongodb_databases', + name, + database.get('hostname', 'localhost'), + database.get('port'), + ) + ) + dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, @@ -82,6 +92,9 @@ def dump_data_sources( ) if not dry_run: + dump.write_data_source_dumps_metadata( + borgmatic_runtime_directory, 'mongodb_databases', dumps_metadata + ) patterns.append( borgmatic.borg.pattern.Pattern( os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 579bf869..ea4aae29 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -188,11 +188,12 @@ def dump_data_sources( ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] + dumps_metadata = [] logger.info(f'Dumping MySQL databases{dry_run_label}') + dump_path = make_dump_path(borgmatic_runtime_directory) for database in databases: - dump_path = make_dump_path(borgmatic_runtime_directory) username = borgmatic.hooks.credential.parse.resolve_credential( database.get('username'), config, @@ -225,9 +226,17 @@ def dump_data_sources( raise ValueError('Cannot find any MySQL databases to dump.') if database['name'] == 'all' and database.get('format'): - for dump_name in dump_database_names: + for database_name in dump_database_names: + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'mysql_databases', + database_name, + database.get('hostname', 'localhost'), + database.get('port'), + ) + ) renamed_database = copy.copy(database) - renamed_database['name'] = dump_name + renamed_database['name'] = database_name processes.append( execute_dump_command( renamed_database, @@ -235,13 +244,21 @@ def dump_data_sources( username, password, dump_path, - (dump_name,), + (database_name,), environment, dry_run, dry_run_label, ), ) else: + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'mysql_databases', + database['name'], + database.get('hostname', 'localhost'), + database.get('port'), + ) + ) processes.append( execute_dump_command( database, @@ -257,6 +274,9 @@ def dump_data_sources( ) if not dry_run: + dump.write_data_source_dumps_metadata( + borgmatic_runtime_directory, 'mysql_databases', dumps_metadata + ) patterns.append( borgmatic.borg.pattern.Pattern( os.path.join(borgmatic_runtime_directory, 'mysql_databases'), diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 123b79a2..f7eac3a2 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -150,12 +150,13 @@ def dump_data_sources( ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] + dumps_metadata = [] logger.info(f'Dumping PostgreSQL databases{dry_run_label}') + dump_path = make_dump_path(borgmatic_runtime_directory) for database in databases: environment = make_environment(database, config) - dump_path = make_dump_path(borgmatic_runtime_directory) dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: @@ -165,6 +166,14 @@ def dump_data_sources( raise ValueError('Cannot find any PostgreSQL databases to dump.') for database_name in dump_database_names: + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'postgresql_databases', + database_name, + database.get('hostname', 'localhost'), + database.get('port'), + ) + ) dump_format = database.get('format', None if database_name == 'all' else 'custom') compression = database.get('compression') default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' @@ -178,6 +187,7 @@ def dump_data_sources( database.get('hostname'), database.get('port'), ) + if os.path.exists(dump_filename): logger.warning( f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}', @@ -247,6 +257,9 @@ def dump_data_sources( ) if not dry_run: + dump.write_data_source_dumps_metadata( + borgmatic_runtime_directory, 'postgresql_databases', dumps_metadata + ) patterns.append( borgmatic.borg.pattern.Pattern( os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 9ac6f2d3..28555e5b 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -49,11 +49,18 @@ def dump_data_sources( ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] + dumps_metadata = [] logger.info(f'Dumping SQLite databases{dry_run_label}') for database in databases: database_path = database['path'] + dumps_metadata.append( + borgmatic.actions.restore.Dump( + 'sqlite_databases', + database['name'], + ) + ) if database['name'] == 'all': logger.warning('The "all" database name has no meaning for SQLite databases') @@ -95,6 +102,9 @@ def dump_data_sources( ) if not dry_run: + dump.write_data_source_dumps_metadata( + borgmatic_runtime_directory, 'sqlite_databases', dumps_metadata + ) patterns.append( borgmatic.borg.pattern.Pattern( os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index d0df4811..9941fdb3 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -462,27 +462,118 @@ def test_restore_single_dump_with_directory_dump_and_dry_run_skips_directory_mov ) -def test_collect_dumps_from_archive_parses_archive_paths(): - flexmock(module.borgmatic.config.paths).should_receive( - 'get_borgmatic_source_directory', - ).and_return('/root/.borgmatic') +def test_collect_dumps_from_archive_with_dumps_metadata_parses_it(): flexmock(module.borgmatic.hooks.data_source.dump).should_receive( 'make_data_source_dump_path', ).and_return('') + flexmock(module.borgmatic.config.paths).should_receive( + 'make_runtime_directory_glob' + ).and_return('') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( [ - 'borgmatic/postgresql_databases/localhost/foo', - 'borgmatic/postgresql_databases/host:1234/bar', - 'borgmatic/mysql_databases/localhost/quux', + 'borgmatic/postgresql_databases/dumps.json', + 'borgmatic/mysql_databases/dumps.json', ], ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return( + flexmock(stdout=flexmock(read=lambda: b'')) + ) + dumps_metadata = [ + module.Dump('postgresql_databases', 'foo'), + module.Dump('postgresql_databases', 'bar', 'host', 1234), + module.Dump('mysql_databases', 'quux'), + ] + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'parse_data_source_dumps_metadata' + ).and_return(dumps_metadata) + flexmock(module.borgmatic.config.paths).should_receive('get_borgmatic_source_directory').never() archive_dumps = module.collect_dumps_from_archive( repository={'path': 'repo'}, archive='archive', config={}, local_borg_version=flexmock(), - global_arguments=flexmock(log_json=False), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path=flexmock(), + remote_path=flexmock(), + borgmatic_runtime_directory='/run/borgmatic', + ) + + assert archive_dumps == set(dumps_metadata) + + +def test_collect_dumps_from_archive_with_empty_dumps_metadata_path_falls_back_to_parsing_archive_paths(): + flexmock(module.borgmatic.config.paths).should_receive( + 'make_runtime_directory_glob' + ).and_return('') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [''] + ).and_return( + [ + 'borgmatic/postgresql_databases/localhost/foo', + 'borgmatic/postgresql_databases/host:1234/bar', + 'borgmatic/mysql_databases/localhost/quux', + ], + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never() + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'parse_data_source_dumps_metadata' + ).never() + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory', + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'make_data_source_dump_path', + ).and_return('') + + archive_dumps = module.collect_dumps_from_archive( + repository={'path': 'repo'}, + archive='archive', + config={}, + local_borg_version=flexmock(), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path=flexmock(), + remote_path=flexmock(), + borgmatic_runtime_directory='/run/borgmatic', + ) + + assert archive_dumps == { + module.Dump('postgresql_databases', 'foo'), + module.Dump('postgresql_databases', 'bar', 'host', 1234), + module.Dump('mysql_databases', 'quux'), + } + + +def test_collect_dumps_from_archive_without_dumps_metadata_falls_back_to_parsing_archive_paths(): + flexmock(module.borgmatic.config.paths).should_receive( + 'make_runtime_directory_glob' + ).and_return('') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [] + ).and_return( + [ + 'borgmatic/postgresql_databases/localhost/foo', + 'borgmatic/postgresql_databases/host:1234/bar', + 'borgmatic/mysql_databases/localhost/quux', + ], + ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never() + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'parse_data_source_dumps_metadata' + ).never() + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory', + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'make_data_source_dump_path', + ).and_return('') + + archive_dumps = module.collect_dumps_from_archive( + repository={'path': 'repo'}, + archive='archive', + config={}, + local_borg_version=flexmock(), + global_arguments=flexmock(dry_run=False, log_json=False), local_path=flexmock(), remote_path=flexmock(), borgmatic_runtime_directory='/run/borgmatic', @@ -497,12 +588,11 @@ def test_collect_dumps_from_archive_parses_archive_paths(): def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_directories(): flexmock(module.borgmatic.config.paths).should_receive( - 'get_borgmatic_source_directory', - ).and_return('/root/.borgmatic') - flexmock(module.borgmatic.hooks.data_source.dump).should_receive( - 'make_data_source_dump_path', + 'make_runtime_directory_glob' ).and_return('') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [] + ).and_return( [ 'borgmatic/postgresql_databases/localhost/foo', '.borgmatic/postgresql_databases/localhost/bar', @@ -510,6 +600,16 @@ def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_dir '/var/run/0/borgmatic/mysql_databases/localhost/quux', ], ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never() + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'parse_data_source_dumps_metadata' + ).never() + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory', + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'make_data_source_dump_path', + ).and_return('') archive_dumps = module.collect_dumps_from_archive( repository={'path': 'repo'}, @@ -532,17 +632,26 @@ def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_dir def test_collect_dumps_from_archive_parses_directory_format_archive_paths(): flexmock(module.borgmatic.config.paths).should_receive( - 'get_borgmatic_source_directory', - ).and_return('/root/.borgmatic') - flexmock(module.borgmatic.hooks.data_source.dump).should_receive( - 'make_data_source_dump_path', + 'make_runtime_directory_glob' ).and_return('') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [] + ).and_return( [ 'borgmatic/postgresql_databases/localhost/foo/table1', 'borgmatic/postgresql_databases/localhost/foo/table2', ], ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never() + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'parse_data_source_dumps_metadata' + ).never() + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory', + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'make_data_source_dump_path', + ).and_return('') archive_dumps = module.collect_dumps_from_archive( repository={'path': 'repo'}, @@ -562,12 +671,11 @@ def test_collect_dumps_from_archive_parses_directory_format_archive_paths(): def test_collect_dumps_from_archive_skips_bad_archive_paths_or_bad_path_components(): flexmock(module.borgmatic.config.paths).should_receive( - 'get_borgmatic_source_directory', - ).and_return('/root/.borgmatic') - flexmock(module.borgmatic.hooks.data_source.dump).should_receive( - 'make_data_source_dump_path', + 'make_runtime_directory_glob' ).and_return('') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + [] + ).and_return( [ 'borgmatic/postgresql_databases/localhost/foo', 'borgmatic/postgresql_databases/localhost:abcd/bar', @@ -576,6 +684,16 @@ def test_collect_dumps_from_archive_skips_bad_archive_paths_or_bad_path_componen '', ], ) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never() + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'parse_data_source_dumps_metadata' + ).never() + flexmock(module.borgmatic.config.paths).should_receive( + 'get_borgmatic_source_directory', + ).and_return('/root/.borgmatic') + flexmock(module.borgmatic.hooks.data_source.dump).should_receive( + 'make_data_source_dump_path', + ).and_return('') archive_dumps = module.collect_dumps_from_archive( repository={'path': 'repo'}, diff --git a/tests/unit/hooks/data_source/test_dump.py b/tests/unit/hooks/data_source/test_dump.py index d1b48ede..9aa35884 100644 --- a/tests/unit/hooks/data_source/test_dump.py +++ b/tests/unit/hooks/data_source/test_dump.py @@ -1,3 +1,6 @@ +import io +import sys + import pytest from flexmock import flexmock @@ -31,6 +34,75 @@ def test_make_data_source_dump_filename_with_invalid_name_raises(): module.make_data_source_dump_filename('databases', 'invalid/name') +def test_write_data_source_dumps_metadata_writes_json_to_file(): + dumps_metadata = [ + module.borgmatic.actions.restore.Dump('databases', 'foo'), + module.borgmatic.actions.restore.Dump('databases', 'bar'), + ] + dumps_stream = io.StringIO('password') + dumps_stream.name = '/run/borgmatic/databases/dumps.json' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args(dumps_stream.name, 'w', encoding='utf-8').and_return( + dumps_stream + ) + flexmock(dumps_stream).should_receive('close') # Prevent close() so getvalue() below works. + + module.write_data_source_dumps_metadata('/run/borgmatic', 'databases', dumps_metadata) + + assert ( + dumps_stream.getvalue() + == '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "port": null}, {"data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "port": null}]}' + ) + + +def test_write_data_source_dumps_metadata_with_operating_system_error_raises(): + dumps_metadata = [ + module.borgmatic.actions.restore.Dump('databases', 'foo'), + module.borgmatic.actions.restore.Dump('databases', 'bar'), + ] + dumps_stream = io.StringIO('password') + dumps_stream.name = '/run/borgmatic/databases/dumps.json' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args(dumps_stream.name, 'w', encoding='utf-8').and_raise( + OSError + ) + + with pytest.raises(ValueError): + module.write_data_source_dumps_metadata('/run/borgmatic', 'databases', dumps_metadata) + + +def test_parse_data_source_dumps_metadata_converts_json_to_dump_instances(): + dumps_json = '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "port": null}, {"data_source_name": "bar", "hook_name": "databases", "hostname": "example.org", "port": 1234}]}' + + assert module.parse_data_source_dumps_metadata( + dumps_json, 'borgmatic/databases/dumps.json' + ) == ( + module.borgmatic.actions.restore.Dump('databases', 'foo'), + module.borgmatic.actions.restore.Dump('databases', 'bar', 'example.org', 1234), + ) + + +def test_parse_data_source_dumps_metadata_with_invalid_json_raises(): + with pytest.raises(ValueError): + module.parse_data_source_dumps_metadata('[{', 'borgmatic/databases/dumps.json') + + +def test_parse_data_source_dumps_metadata_with_unknown_keys_raises(): + dumps_json = ( + '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "wtf": "is this"}]}' + ) + + with pytest.raises(ValueError): + module.parse_data_source_dumps_metadata(dumps_json, 'borgmatic/databases/dumps.json') + + +def test_parse_data_source_dumps_metadata_with_missing_dumps_key_raises(): + dumps_json = '{"not": "what we are looking for"}' + + with pytest.raises(ValueError): + module.parse_data_source_dumps_metadata(dumps_json, 'borgmatic/databases/dumps.json') + + def test_create_parent_directory_for_dump_does_not_raise(): flexmock(module.os).should_receive('makedirs') diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index f2ece606..922d5e1c 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -349,6 +349,15 @@ def test_dump_data_sources_dumps_each_database(): dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'), + ], + ).once() + assert ( module.dump_data_sources( databases, @@ -390,6 +399,13 @@ def test_dump_data_sources_dumps_with_password(): dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( [database], @@ -434,6 +450,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( [database], @@ -465,6 +488,13 @@ def test_dump_data_sources_dumps_all_databases_at_once(): dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'all'), + ], + ).once() assert module.dump_data_sources( databases, @@ -499,6 +529,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'), + ], + ).once() + assert ( module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index d09dff60..98f5ae2e 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -39,6 +39,15 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mongodb_databases', 'bar'), + ], + ).once() + assert ( module.dump_data_sources( databases, @@ -60,6 +69,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): ).and_return('databases/localhost/bar') flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never() assert ( module.dump_data_sources( @@ -75,7 +85,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): - databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 27018}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -89,7 +99,7 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): '--host', 'database.example.org', '--port', - '5433', + '27018', '--db', 'foo', '--archive', @@ -99,6 +109,15 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump( + 'mongodb_databases', 'foo', 'database.example.org', 27018 + ), + ], + ).once() assert module.dump_data_sources( databases, @@ -150,6 +169,13 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( databases, @@ -174,6 +200,13 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): ('mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'), shell=True, ).and_return(flexmock()).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ], + ).once() assert ( module.dump_data_sources( @@ -210,6 +243,13 @@ def test_dump_data_sources_runs_mongodump_with_options(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( databases, @@ -235,6 +275,13 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'all'), + ], + ).once() assert module.dump_data_sources( databases, @@ -308,7 +355,7 @@ def test_restore_data_source_dump_runs_mongorestore(): def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): hook_config = [ - {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None}, + {'name': 'foo', 'hostname': 'database.example.org', 'port': 27018, 'schemas': None}, ] extract_process = flexmock(stdout=flexmock()) @@ -325,7 +372,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): '--host', 'database.example.org', '--port', - '5433', + '27018', ], processes=[extract_process], output_log_level=logging.DEBUG, @@ -708,6 +755,13 @@ def test_dump_data_sources_uses_custom_mongodump_command(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 262d9c44..f9cbe099 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -232,6 +232,15 @@ def test_dump_data_sources_dumps_each_database(): dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'), + ], + ).once() + assert ( module.dump_data_sources( databases, @@ -273,6 +282,13 @@ def test_dump_data_sources_dumps_with_password(): dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( [database], @@ -317,6 +333,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( [database], @@ -349,6 +372,14 @@ def test_dump_data_sources_dumps_all_databases_at_once(): dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'all'), + ], + ).once() + assert module.dump_data_sources( databases, {}, @@ -382,6 +413,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'), + ], + ).once() + assert ( module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 15c65d48..22f7ee4c 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -273,6 +273,14 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'), + ], + ).once() assert ( module.dump_data_sources( @@ -336,6 +344,14 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'), + ], + ).once() assert ( module.dump_data_sources( @@ -366,6 +382,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never() assert ( module.dump_data_sources( @@ -415,6 +432,15 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump( + 'postgresql_databases', 'foo', 'database.example.org', 5433 + ), + ], + ).once() assert module.dump_data_sources( databases, @@ -461,6 +487,13 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( databases, @@ -507,6 +540,13 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( databases, @@ -548,6 +588,13 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): shell=True, environment={'PGSSLMODE': 'disable'}, ).and_return(flexmock()).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ], + ).once() assert ( module.dump_data_sources( @@ -595,6 +642,13 @@ def test_dump_data_sources_runs_pg_dump_with_string_compression(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(processes[0]).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ], + ).once() assert ( module.dump_data_sources( @@ -642,6 +696,13 @@ def test_dump_data_sources_runs_pg_dump_with_integer_compression(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(processes[0]).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ], + ).once() assert ( module.dump_data_sources( @@ -688,6 +749,13 @@ def test_dump_data_sources_runs_pg_dump_with_options(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( databases, @@ -720,6 +788,13 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'all'), + ], + ).once() assert module.dump_data_sources( databases, @@ -764,6 +839,13 @@ def test_dump_data_sources_runs_non_default_pg_dump(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ], + ).once() assert module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 4c532266..3531c975 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -26,6 +26,13 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database'), + ], + ).once() assert ( module.dump_data_sources( @@ -56,6 +63,14 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module).should_receive('execute_command').and_return(processes[0]).and_return( processes[1], ) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database2'), + ], + ).once() assert ( module.dump_data_sources( @@ -93,6 +108,13 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): shell=True, run_to_completion=False, ).and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + ], + ).once() assert ( module.dump_data_sources( @@ -135,6 +157,13 @@ def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_ge shell=True, run_to_completion=False, ).and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + ], + ).once() assert ( module.dump_data_sources( @@ -163,6 +192,13 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + ], + ).once() assert ( module.dump_data_sources( @@ -193,6 +229,13 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'all'), + ], + ).once() assert ( module.dump_data_sources( @@ -217,6 +260,7 @@ def test_dump_data_sources_does_not_dump_if_dry_run(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never() assert ( module.dump_data_sources(