diff --git a/NEWS b/NEWS index 3dfa3909..db81da6f 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,7 @@ use. * #1126: Create LVM snapshots as read-write to avoid an error when snapshotting ext4 filesystems with orphaned files that need recovery. - * #1136: For all database hooks, record metadata on the dumps contained within an archive. + * #1136: For all database hooks, record metadata about the dumps contained within an archive. * #1139: Set "borgmatic" as the user agent when connecting to monitoring services. * When running tests, use Ruff for faster and more comprehensive code linting and formatting, replacing Flake8, Black, isort, etc. diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index ea8aa61b..50d66ea6 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -294,14 +294,13 @@ def dump_data_sources( raise ValueError('Cannot find any MariaDB databases to dump.') - if database['name'] == 'all' and database.get('format'): for database_name in dump_database_names: dumps_metadata.append( borgmatic.actions.restore.Dump( 'mariadb_databases', database_name, - database.get('hostname'), + database.get('hostname', 'localhost'), database.get('port'), ) ) @@ -325,7 +324,7 @@ def dump_data_sources( borgmatic.actions.restore.Dump( 'mariadb_databases', database['name'], - database.get('hostname'), + database.get('hostname', 'localhost'), database.get('port'), ) ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 48ba4184..3a249aaa 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -61,7 +61,7 @@ def dump_data_sources( borgmatic.actions.restore.Dump( 'mongodb_databases', name, - database.get('hostname'), + database.get('hostname', 'localhost'), database.get('port'), ) ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 50332906..ea4aae29 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -225,14 +225,13 @@ def dump_data_sources( raise ValueError('Cannot find any MySQL databases to dump.') - if database['name'] == 'all' and database.get('format'): for database_name in dump_database_names: dumps_metadata.append( borgmatic.actions.restore.Dump( 'mysql_databases', database_name, - database.get('hostname'), + database.get('hostname', 'localhost'), database.get('port'), ) ) @@ -256,7 +255,7 @@ def dump_data_sources( borgmatic.actions.restore.Dump( 'mysql_databases', database['name'], - database.get('hostname'), + database.get('hostname', 'localhost'), database.get('port'), ) ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index c4fc55bf..f7eac3a2 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -170,7 +170,7 @@ def dump_data_sources( borgmatic.actions.restore.Dump( 'postgresql_databases', database_name, - database.get('hostname'), + database.get('hostname', 'localhost'), database.get('port'), ) ) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index f2ece606..184a0ff9 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -349,6 +349,15 @@ def test_dump_data_sources_dumps_each_database(): dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'), + ] + ).once() + assert ( module.dump_data_sources( databases, @@ -390,6 +399,13 @@ def test_dump_data_sources_dumps_with_password(): dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( [database], @@ -434,6 +450,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( [database], @@ -465,6 +488,13 @@ def test_dump_data_sources_dumps_all_databases_at_once(): dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'all'), + ] + ).once() assert module.dump_data_sources( databases, @@ -499,6 +529,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mariadb_databases', + [ + module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'), + ] + ).once() + assert ( module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index d09dff60..8ac97c11 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -39,6 +39,15 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mongodb_databases', 'bar'), + ] + ).once() + assert ( module.dump_data_sources( databases, @@ -60,6 +69,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): ).and_return('databases/localhost/bar') flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never() assert ( module.dump_data_sources( @@ -75,7 +85,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): - databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] + databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 27018}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -89,7 +99,7 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): '--host', 'database.example.org', '--port', - '5433', + '27018', '--db', 'foo', '--archive', @@ -99,6 +109,13 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo', 'database.example.org', 27018), + ] + ).once() assert module.dump_data_sources( databases, @@ -150,6 +167,13 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( databases, @@ -174,6 +198,13 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): ('mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'), shell=True, ).and_return(flexmock()).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ] + ).once() assert ( module.dump_data_sources( @@ -210,6 +241,13 @@ def test_dump_data_sources_runs_mongodump_with_options(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( databases, @@ -235,6 +273,13 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'all'), + ] + ).once() assert module.dump_data_sources( databases, @@ -308,7 +353,7 @@ def test_restore_data_source_dump_runs_mongorestore(): def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): hook_config = [ - {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None}, + {'name': 'foo', 'hostname': 'database.example.org', 'port': 27018, 'schemas': None}, ] extract_process = flexmock(stdout=flexmock()) @@ -325,7 +370,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): '--host', 'database.example.org', '--port', - '5433', + '27018', ], processes=[extract_process], output_log_level=logging.DEBUG, @@ -708,6 +753,13 @@ def test_dump_data_sources_uses_custom_mongodump_command(): shell=True, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mongodb_databases', + [ + module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 262d9c44..904282c2 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -232,6 +232,15 @@ def test_dump_data_sources_dumps_each_database(): dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'), + ] + ).once() + assert ( module.dump_data_sources( databases, @@ -273,6 +282,13 @@ def test_dump_data_sources_dumps_with_password(): dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( [database], @@ -317,6 +333,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass dry_run=object, dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( [database], @@ -349,6 +372,14 @@ def test_dump_data_sources_dumps_all_databases_at_once(): dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'all'), + ] + ).once() + assert module.dump_data_sources( databases, {}, @@ -382,6 +413,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured dry_run_label=object, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'mysql_databases', + [ + module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'), + ] + ).once() + assert ( module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 15c65d48..0971f6b9 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -273,6 +273,14 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'), + ] + ).once() assert ( module.dump_data_sources( @@ -336,6 +344,14 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'), + ] + ).once() assert ( module.dump_data_sources( @@ -366,6 +382,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never() assert ( module.dump_data_sources( @@ -415,6 +432,13 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo', 'database.example.org', 5433), + ] + ).once() assert module.dump_data_sources( databases, @@ -461,6 +485,13 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( databases, @@ -507,6 +538,13 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( databases, @@ -548,6 +586,13 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): shell=True, environment={'PGSSLMODE': 'disable'}, ).and_return(flexmock()).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ] + ).once() assert ( module.dump_data_sources( @@ -595,6 +640,13 @@ def test_dump_data_sources_runs_pg_dump_with_string_compression(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(processes[0]).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ] + ).once() assert ( module.dump_data_sources( @@ -642,6 +694,13 @@ def test_dump_data_sources_runs_pg_dump_with_integer_compression(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(processes[0]).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ] + ).once() assert ( module.dump_data_sources( @@ -688,6 +747,13 @@ def test_dump_data_sources_runs_pg_dump_with_options(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( databases, @@ -720,6 +786,13 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'all'), + ] + ).once() assert module.dump_data_sources( databases, @@ -764,6 +837,13 @@ def test_dump_data_sources_runs_non_default_pg_dump(): environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'postgresql_databases', + [ + module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'), + ] + ).once() assert module.dump_data_sources( databases, diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 4c532266..e1dcd510 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -26,6 +26,13 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database'), + ] + ).once() assert ( module.dump_data_sources( @@ -56,6 +63,14 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module).should_receive('execute_command').and_return(processes[0]).and_return( processes[1], ) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database2'), + ] + ).once() assert ( module.dump_data_sources( @@ -93,6 +108,13 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): shell=True, run_to_completion=False, ).and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + ] + ).once() assert ( module.dump_data_sources( @@ -135,6 +157,13 @@ def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_ge shell=True, run_to_completion=False, ).and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + ] + ).once() assert ( module.dump_data_sources( @@ -163,6 +192,13 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'), + ] + ).once() assert ( module.dump_data_sources( @@ -193,6 +229,13 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').and_return(processes[0]) + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args( + '/run/borgmatic', + 'sqlite_databases', + [ + module.borgmatic.actions.restore.Dump('sqlite_databases', 'all'), + ] + ).once() assert ( module.dump_data_sources( @@ -217,6 +260,7 @@ def test_dump_data_sources_does_not_dump_if_dry_run(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() + flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never() assert ( module.dump_data_sources(