Fix "restore" action and broken end-to-end tests.

This commit is contained in:
Dan Helfman
2026-01-14 12:54:36 -08:00
parent d354cf0942
commit 762beb46c2
14 changed files with 101 additions and 48 deletions
+8 -6
View File
@@ -277,7 +277,7 @@ def collect_dumps_from_archive(
dumps_from_archive = {} # Use a dict as an ordered set.
# There is (at most) one dump metadata file per data source hook. Load each.
for dumps_metadata_path in borgmatic.borg.list.capture_archive_listing(
for dumps_metadata_entry in borgmatic.borg.list.capture_archive_listing(
repository,
archive,
config,
@@ -300,7 +300,7 @@ def collect_dumps_from_archive(
local_path=local_path,
remote_path=remote_path,
):
if not dumps_metadata_path:
if not dumps_metadata_entry.get('path'):
continue
for dump in borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata(
@@ -308,7 +308,7 @@ def collect_dumps_from_archive(
global_arguments.dry_run,
repository,
archive,
[dumps_metadata_path],
[dumps_metadata_entry['path']],
config,
local_borg_version,
global_arguments,
@@ -318,7 +318,7 @@ def collect_dumps_from_archive(
)
.stdout.read()
.decode(),
dumps_metadata_path,
dumps_metadata_entry['path'],
):
dumps_from_archive[dump] = None
@@ -338,7 +338,7 @@ def collect_dumps_from_archive(
# Probe for the data source dumps in multiple locations, as the default location has moved to
# the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
# we still want to support reading dumps from previously created archives as well.
dump_paths = borgmatic.borg.list.capture_archive_listing(
dump_entries = borgmatic.borg.list.capture_archive_listing(
repository,
archive,
config,
@@ -360,7 +360,9 @@ def collect_dumps_from_archive(
remote_path=remote_path,
)
for dump_path in dump_paths:
for dump_entry in dump_entries:
dump_path = dump_entry.get('path')
if not dump_path:
continue
+1 -5
View File
@@ -135,11 +135,7 @@ def extract_archive(
+ (('--remote-path', remote_path) if remote_path else ())
+ numeric_ids_flags
+ (('--umask', str(umask)) if umask else ())
+ (
('--log-json',)
if (config.get('log_json') or (not config.get('progress') and not extract_to_stdout))
else ()
)
+ (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+1
View File
@@ -524,5 +524,6 @@ def restore_data_source_dump(
input_file=extract_process.stdout,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=config.get('local_path', 'borg'),
)
)
+1
View File
@@ -296,6 +296,7 @@ def restore_data_source_dump(
output_log_level=logging.DEBUG,
input_file=extract_process.stdout if extract_process else None,
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=config.get('local_path', 'borg'),
)
)
+1
View File
@@ -461,5 +461,6 @@ def restore_data_source_dump(
input_file=extract_process.stdout,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=config.get('local_path', 'borg'),
)
)
@@ -442,6 +442,7 @@ def restore_data_source_dump(
input_file=extract_process.stdout if extract_process else None,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=config.get('local_path', 'borg'),
)
)
execute_command(
+2
View File
@@ -215,6 +215,7 @@ def restore_data_source_dump(
for part in shlex.split(data_source.get('sqlite_restore_command') or 'sqlite3')
)
restore_command = (*sqlite_restore_command, '-bail', shlex.quote(database_path))
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
tuple(
@@ -224,5 +225,6 @@ def restore_data_source_dump(
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=config.get('local_path', 'borg'),
)
)
+36 -36
View File
@@ -513,10 +513,10 @@ def test_collect_dumps_from_archive_with_dumps_metadata_parses_it():
'make_runtime_directory_glob'
).and_return('')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
[
'borgmatic/postgresql_databases/dumps.json',
'borgmatic/mysql_databases/dumps.json',
],
(
{'path': 'borgmatic/postgresql_databases/dumps.json'},
{'path': 'borgmatic/mysql_databases/dumps.json'},
),
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
flexmock(stdout=flexmock(read=lambda: b''))
@@ -550,13 +550,13 @@ def test_collect_dumps_from_archive_with_empty_dumps_metadata_path_falls_back_to
'make_runtime_directory_glob'
).and_return('')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['']
({'path': ''},)
).and_return(
[
'borgmatic/postgresql_databases/localhost/foo',
'borgmatic/postgresql_databases/host:1234/bar',
'borgmatic/mysql_databases/localhost/quux',
],
(
{'path': 'borgmatic/postgresql_databases/localhost/foo'},
{'path': 'borgmatic/postgresql_databases/host:1234/bar'},
{'path': 'borgmatic/mysql_databases/localhost/quux'},
),
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
@@ -592,13 +592,13 @@ def test_collect_dumps_from_archive_without_dumps_metadata_falls_back_to_parsing
'make_runtime_directory_glob'
).and_return('')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
[]
()
).and_return(
[
'borgmatic/postgresql_databases/localhost/foo',
'borgmatic/postgresql_databases/host:1234/bar',
'borgmatic/mysql_databases/localhost/quux',
],
(
{'path': 'borgmatic/postgresql_databases/localhost/foo'},
{'path': 'borgmatic/postgresql_databases/host:1234/bar'},
{'path': 'borgmatic/mysql_databases/localhost/quux'},
),
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
@@ -634,14 +634,14 @@ def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_dir
'make_runtime_directory_glob'
).and_return('')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
[]
()
).and_return(
[
'borgmatic/postgresql_databases/localhost/foo',
'.borgmatic/postgresql_databases/localhost/bar',
'/root/.borgmatic/postgresql_databases/localhost/baz',
'/var/run/0/borgmatic/mysql_databases/localhost/quux',
],
(
{'path': 'borgmatic/postgresql_databases/localhost/foo'},
{'path': '.borgmatic/postgresql_databases/localhost/bar'},
{'path': '/root/.borgmatic/postgresql_databases/localhost/baz'},
{'path': '/var/run/0/borgmatic/mysql_databases/localhost/quux'},
),
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
@@ -678,12 +678,12 @@ def test_collect_dumps_from_archive_parses_directory_format_archive_paths():
'make_runtime_directory_glob'
).and_return('')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
[]
()
).and_return(
[
'borgmatic/postgresql_databases/localhost/foo/table1',
'borgmatic/postgresql_databases/localhost/foo/table2',
],
(
{'path': 'borgmatic/postgresql_databases/localhost/foo/table1'},
{'path': 'borgmatic/postgresql_databases/localhost/foo/table2'},
),
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
@@ -715,15 +715,15 @@ def test_collect_dumps_from_archive_skips_bad_archive_paths_or_bad_path_componen
'make_runtime_directory_glob'
).and_return('')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
[]
()
).and_return(
[
'borgmatic/postgresql_databases/localhost/foo',
'borgmatic/postgresql_databases/localhost:abcd/bar',
'borgmatic/invalid',
'invalid/as/well',
'',
],
(
{'path': 'borgmatic/postgresql_databases/localhost/foo'},
{'path': 'borgmatic/postgresql_databases/localhost:abcd/bar'},
{'path': 'borgmatic/invalid'},
{'path': 'invalid/as/well'},
{'path': ''},
)
)
flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').never()
flexmock(module.borgmatic.hooks.data_source.dump).should_receive(
+1 -1
View File
@@ -715,7 +715,7 @@ def test_extract_archive_calls_borg_with_extract_to_stdout_returns_process():
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module).should_receive('execute_command').with_args(
('borg', 'extract', '--stdout', 'repo::archive'),
('borg', 'extract', '--log-json', '--stdout', 'repo::archive'),
output_file=module.subprocess.PIPE,
run_to_completion=False,
environment=None,
@@ -1353,6 +1353,7 @@ def test_restore_data_source_dump_runs_mariadb_to_restore():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1394,6 +1395,7 @@ def test_restore_data_source_dump_runs_mariadb_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1437,6 +1439,7 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1489,6 +1492,7 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1537,6 +1541,7 @@ def test_restore_data_source_dump_runs_mariadb_with_socket_path():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1584,6 +1589,7 @@ def test_restore_data_source_dump_runs_mariadb_with_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1631,6 +1637,7 @@ def test_restore_data_source_dump_runs_mariadb_without_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1674,6 +1681,7 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1722,6 +1730,7 @@ def test_restore_data_source_with_environment_password_transport_skips_defaults_
input_file=extract_process.stdout,
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1787,6 +1796,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1857,6 +1867,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -394,6 +394,7 @@ def test_restore_data_source_dump_runs_mongorestore():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -438,6 +439,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -493,6 +495,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password()
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -556,6 +559,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -619,6 +623,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -653,6 +658,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_options():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -695,6 +701,7 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -729,6 +736,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -788,6 +796,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk():
output_log_level=logging.DEBUG,
input_file=None,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -901,6 +910,7 @@ def test_restore_data_source_dump_uses_custom_mongorestore_command():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1235,6 +1235,7 @@ def test_restore_data_source_dump_runs_mysql_to_restore():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1276,6 +1277,7 @@ def test_restore_data_source_dump_runs_mysql_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1317,6 +1319,7 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1369,6 +1372,7 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1421,6 +1425,7 @@ def test_restore_data_source_dump_runs_mysql_with_socket_path():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1468,6 +1473,7 @@ def test_restore_data_source_dump_runs_mysql_with_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1515,6 +1521,7 @@ def test_restore_data_source_dump_runs_mysql_without_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1558,6 +1565,7 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1610,6 +1618,7 @@ def test_restore_data_source_with_environment_password_transport_skips_defaults_
input_file=extract_process.stdout,
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1677,6 +1686,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -1749,6 +1759,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
module.restore_data_source_dump(
@@ -995,6 +995,7 @@ def test_restore_data_source_dump_runs_pg_restore():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1059,6 +1060,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1127,6 +1129,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password():
input_file=extract_process.stdout,
environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1208,6 +1211,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
input_file=extract_process.stdout,
environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1293,6 +1297,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
input_file=extract_process.stdout,
environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1365,6 +1370,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1420,6 +1426,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'),
@@ -1461,6 +1468,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1531,6 +1539,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1621,6 +1630,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk():
input_file=None,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -1683,6 +1693,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas():
input_file=None,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
@@ -341,6 +341,7 @@ def test_restore_data_source_dump_restores_database():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -379,6 +380,7 @@ def test_restore_data_source_dump_runs_non_default_sqlite_restores_database():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -415,6 +417,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -451,6 +454,7 @@ def test_restore_data_source_dump_runs_non_default_sqlite_with_connection_params
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -489,6 +493,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -526,6 +531,7 @@ def test_restore_data_source_dump_runs_non_default_sqlite_without_connection_par
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
borg_local_path='borg',
).and_yield().once()
flexmock(module.os).should_receive('remove').once()