peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-22 02:03:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Finalize container support

Browse Source
This commit is contained in:
Florian Apolloner
2025-09-15 22:16:49 +02:00
parent 56a3f6d854
commit 0e90087dc4
12 changed files with 193 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
borgmatic/actions/restore.py
+12 -7
View File
@@ -22,8 +22,8 @@ UNSPECIFIED = object()
Dump = collections.namedtuple(
'Dump',
('hook_name', 'data_source_name', 'hostname', 'port', 'label'),
defaults=('localhost', None, None),
('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'),
defaults=('localhost', None, None, None),
)
@@ -33,7 +33,7 @@ def dumps_match(first, second, default_port=None):
indicates that the field should match any value. If a default port is given, then consider any
dump having that port to match with a dump having a None port.
'''
# label kinda counts as id, if they match ignore hostname & port
# label kinda counts as an unique id, if they match ignore host/container/port
if first.label not in {None, UNSPECIFIED} and first.label == second.label:
field_list = ('hook_name', 'data_source_name')
else:
@@ -65,15 +65,15 @@ def render_dump_metadata(dump):
'''
label = dump.label or UNSPECIFIED
name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
hostname = dump.hostname or UNSPECIFIED
host = dump.container or dump.hostname or UNSPECIFIED
port = None if dump.port is UNSPECIFIED else dump.port
if label is not UNSPECIFIED:
metadata = f'{name}@{label}'
elif port:
metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}'
else:
metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}'
if dump.hook_name not in {None, UNSPECIFIED}:
return f'{metadata} ({dump.hook_name})'
@@ -110,7 +110,8 @@ def get_configured_data_source(config, restore_dump):
hook_data_source.get('name'),
hook_data_source.get('hostname', 'localhost'),
hook_data_source.get('port'),
hook_data_source.get('label') or hook_data_source.get('container') or UNSPECIFIED,
hook_data_source.get('label') or UNSPECIFIED,
hook_data_source.get('container'),
),
restore_dump,
default_port,
@@ -425,6 +426,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
hostname=restore_arguments.original_hostname or UNSPECIFIED,
port=restore_arguments.original_port,
label=restore_arguments.original_label or UNSPECIFIED,
container=restore_arguments.original_container or UNSPECIFIED,
)
for name in restore_arguments.data_sources or (UNSPECIFIED,)
}
@@ -433,6 +435,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
or restore_arguments.original_hostname
or restore_arguments.original_port
or restore_arguments.original_label
or restore_arguments.original_container
else {
Dump(
hook_name=UNSPECIFIED,
@@ -440,6 +443,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
hostname=UNSPECIFIED,
port=UNSPECIFIED,
label=UNSPECIFIED,
container=UNSPECIFIED,
),
}
)
@@ -585,6 +589,7 @@ def run_restore(
restore_dump.hostname,
restore_dump.port,
restore_dump.label,
restore_dump.container,
),
)
borgmatic/commands/arguments.py
+4
View File
@@ -1499,6 +1499,10 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915
'--original-hostname',
help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
)
restore_group.add_argument(
'--original-container',
help='The container where the dump to restore came from, only necessary if you need to disambiguate dumps',
)
restore_group.add_argument(
'--original-port',
type=int,
borgmatic/hooks/data_source/config.py
+19
View File
@@ -11,6 +11,14 @@ logger = logging.getLogger(__name__)
def resolve_database_option(option, data_source, connection_params=None, restore=False):
'''
Resolves a database option from the given data source configuration dict and
connection parameters dict. If restore is set to True it will consider the
`restore_<option>` instead.
Returns the resolved option or None. Can raise a ValueError if the hostname lookup
results in a container IP check.
'''
# Special case `hostname` since it overlaps with `container`
if option == 'hostname':
return get_hostname_from_config(data_source, connection_params, restore)
@@ -22,6 +30,12 @@ def resolve_database_option(option, data_source, connection_params=None, restore
def get_hostname_from_config(data_source, connection_params=None, restore=False):
'''
Specialisation of `resolve_database_option` to handle the extra complexity of
the hostname option to also handle containers.
Returns a hostname/IP or raises an ValueError if a container IP lookup fails.
'''
# connection params win, full stop
if connection_params:
if container := connection_params.get('container'):
@@ -41,6 +55,11 @@ def get_hostname_from_config(data_source, connection_params=None, restore=False)
def get_ip_from_container(container):
'''
Determine the IP for a given container name via podman and docker.
Returns an IP or raises a ValueError if the lookup fails.
'''
engines = (shutil.which(engine) for engine in ('docker', 'podman'))
engines = [engine for engine in engines if engine]
borgmatic/hooks/data_source/dump.py
+9 -4
View File
@@ -19,7 +19,9 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam
return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
def make_data_source_dump_filename(dump_path, name, hostname=None, port=None, label=None):
def make_data_source_dump_filename(
dump_path, name, hostname=None, port=None, container=None, label=None
):
'''
Based on the given dump directory path, data source name, hostname, and port, return a filename
to use for the data source dump. The hostname defaults to localhost.
@@ -29,9 +31,12 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None, la
if os.path.sep in name:
raise ValueError(f'Invalid data source name {name}')
identifier = (
label if label else (hostname or 'localhost') + ('' if port is None else f':{port}')
)
if container:
hostname = container
elif not hostname:
hostname = 'localhost'
identifier = label if label else hostname + ('' if port is None else f':{port}')
return os.path.join(dump_path, identifier, name)
borgmatic/hooks/data_source/mariadb.py
+6 -1
View File
@@ -180,7 +180,8 @@ def execute_dump_command(
database['name'],
hostname=database.get('hostname'),
port=database.get('port'),
label=database.get('label', database.get('container')),
container=database.get('container'),
label=database.get('label'),
)
if os.path.exists(dump_filename):
@@ -306,6 +307,8 @@ def dump_data_sources(
database_name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
renamed_database = copy.copy(database)
@@ -330,6 +333,8 @@ def dump_data_sources(
database['name'],
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
processes.append(
borgmatic/hooks/data_source/mongodb.py
+6 -2
View File
@@ -64,6 +64,8 @@ def dump_data_sources(
name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
@@ -72,7 +74,8 @@ def dump_data_sources(
name,
hostname=database.get('hostname'),
port=database.get('port'),
label=database.get('label', database.get('container')),
container=database.get('container'),
label=database.get('label'),
)
dump_format = database.get('format', 'archive')
@@ -245,7 +248,8 @@ def restore_data_source_dump(
data_source['name'],
hostname=data_source.get('hostname'),
port=data_source.get('port'),
label=data_source.get('label', data_source.get('container')),
container=data_source.get('container'),
label=data_source.get('label'),
)
restore_command = build_restore_command(
extract_process,
borgmatic/hooks/data_source/mysql.py
+6 -1
View File
@@ -106,7 +106,8 @@ def execute_dump_command(
database['name'],
hostname=database.get('hostname'),
port=database.get('port'),
label=database.get('label', database.get('container')),
container=database.get('container'),
label=database.get('label'),
)
if os.path.exists(dump_filename):
@@ -237,6 +238,8 @@ def dump_data_sources(
database_name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
renamed_database = copy.copy(database)
@@ -261,6 +264,8 @@ def dump_data_sources(
database['name'],
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
processes.append(
borgmatic/hooks/data_source/postgresql.py
+6 -2
View File
@@ -167,6 +167,8 @@ def dump_data_sources(
database_name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
dump_format = database.get('format', None if database_name == 'all' else 'custom')
@@ -181,7 +183,8 @@ def dump_data_sources(
database_name,
hostname=database.get('hostname'),
port=database.get('port'),
label=database.get('label', database.get('container')),
container=database.get('container'),
label=database.get('label'),
)
if os.path.exists(dump_filename):
@@ -351,7 +354,8 @@ def restore_data_source_dump(
data_source['name'],
hostname=hostname,
port=port,
label=data_source.get('label', data_source.get('container')),
container=data_source.get('container'),
label=data_source.get('label'),
)
psql_command = tuple(
shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql')
borgmatic/hooks/data_source/sqlite.py
+1 -2
View File
@@ -57,8 +57,7 @@ def dump_data_sources(
database_path = database['path']
dumps_metadata.append(
borgmatic.actions.restore.Dump(
'sqlite_databases',
database['name'],
'sqlite_databases', database['name'], label=database.get('label')
)
)
tests/end-to-end/hooks/data_source/test_database.py
+11 -7
View File
@@ -117,29 +117,29 @@ encryption_passphrase: "test"
postgresql_databases:
- name: test
hostname: postgresql
container: postgresql
username: postgres
password: test
restore_hostname: postgresql2
restore_container: postgresql2
restore_port: 5433
restore_password: test2
mariadb_databases:
- name: test
hostname: mariadb
container: mariadb
username: root
password: test
restore_hostname: mariadb2
restore_container: mariadb2
restore_port: 3307
restore_username: root
restore_password: test2
mysql_databases:
- name: test
hostname: not-actually-mysql
container: not-actually-mysql
username: root
password: test
mongodb_databases:
- name: test
hostname: mongodb
container: mongodb
username: root
password: test
authentication_database: admin
@@ -273,7 +273,11 @@ postgresql_databases:
def get_connection_params(database, use_restore_options=False):
hostname = (database.get('restore_hostname') if use_restore_options else None) or database.get(
hostname = (
database.get('restore_container', database.get('restore_hostname'))
if use_restore_options
else None
) or database.get(
'container',
database.get('hostname'),
)
tests/unit/actions/test_restore.py
+103 -17
View File
@@ -138,6 +138,12 @@ import borgmatic.actions.restore as module
5432,
True,
),
(
module.Dump('postgresql_databases', 'foo', container='container'),
module.Dump('postgresql_databases', 'foo', container='container'),
5432,
True,
),
),
)
def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values(
@@ -188,6 +194,10 @@ def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values(
module.Dump('postgresql_databases', 'foo', 'host', 1234, 'label'),
'foo@label (postgresql_databases)',
),
(
module.Dump('postgresql_databases', 'foo', container='container'),
'foo@container (postgresql_databases)',
),
(
module.Dump(
module.UNSPECIFIED,
@@ -554,9 +564,9 @@ def test_collect_dumps_from_archive_with_empty_dumps_metadata_path_falls_back_to
)
assert archive_dumps == {
module.Dump('postgresql_databases', 'foo'),
module.Dump('postgresql_databases', 'bar', 'host', 1234),
module.Dump('mysql_databases', 'quux'),
module.Dump('postgresql_databases', 'foo', label='localhost'),
module.Dump('postgresql_databases', 'bar', 'host', 1234, label='host:1234'),
module.Dump('mysql_databases', 'quux', label='localhost'),
}
@@ -737,13 +747,21 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive():
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'bar',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'bar'),
).and_return(True)
@@ -755,6 +773,7 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive():
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -770,7 +789,11 @@ def test_get_dumps_to_restore_raises_for_requested_dumps_missing_from_archive():
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
@@ -784,6 +807,7 @@ def test_get_dumps_to_restore_raises_for_requested_dumps_missing_from_archive():
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
@@ -805,6 +829,7 @@ def test_get_dumps_to_restore_without_requested_dumps_finds_all_archive_dumps():
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
@@ -819,11 +844,23 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_finds_all_archive_dump
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'bar',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'bar'),
).and_return(True)
@@ -835,6 +872,7 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_finds_all_archive_dump
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
@@ -850,13 +888,21 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'bar',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'bar'),
).and_return(True)
@@ -869,6 +915,7 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
@@ -880,13 +927,21 @@ def test_get_dumps_to_restore_raises_for_multiple_matching_dumps_in_archive():
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('mariadb_databases', 'foo'),
).and_return(True)
@@ -900,6 +955,7 @@ def test_get_dumps_to_restore_raises_for_multiple_matching_dumps_in_archive():
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive={
module.Dump('postgresql_databases', 'foo'),
@@ -913,7 +969,11 @@ def test_get_dumps_to_restore_raises_for_all_in_requested_dumps_and_requested_du
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
@@ -927,6 +987,7 @@ def test_get_dumps_to_restore_raises_for_all_in_requested_dumps_and_requested_du
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive={module.Dump('postgresql_databases', 'foo')},
)
@@ -942,7 +1003,11 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
'postgresql_databases', 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
'postgresql_databases',
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
@@ -954,6 +1019,7 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -970,7 +1036,11 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
'postgresql_databases', 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED
'postgresql_databases',
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
@@ -982,6 +1052,7 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -997,7 +1068,13 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'foo', 'host', label=module.UNSPECIFIED),
module.Dump(
'postgresql_databases',
'foo',
'host',
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo', 'host'),
).and_return(True)
@@ -1008,6 +1085,7 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc
original_hostname='host',
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -1023,7 +1101,14 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'foo', 'host', 1234, label=module.UNSPECIFIED),
module.Dump(
'postgresql_databases',
'foo',
'host',
1234,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo', 'host', 1234),
).and_return(True)
@@ -1034,6 +1119,7 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive
original_hostname='host',
original_port=1234,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
tests/unit/hooks/data_source/test_dump.py
+10 -1
View File
@@ -34,6 +34,15 @@ def test_make_data_source_dump_filename_uses_label():
)
def test_make_data_source_dump_filename_uses_container():
assert (
module.make_data_source_dump_filename(
'databases', 'test', 'hostname', 1234, container='container'
)
== 'databases/container:1234/test'
)
def test_make_data_source_dump_filename_without_hostname_defaults_to_localhost():
assert module.make_data_source_dump_filename('databases', 'test') == 'databases/localhost/test'
@@ -60,7 +69,7 @@ def test_write_data_source_dumps_metadata_writes_json_to_file():
assert (
dumps_stream.getvalue()
== '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "port": null}, {"data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "port": null}]}'
== '{"dumps": [{"container": null, "data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "label": null, "port": null}, {"container": null, "data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "label": null, "port": null}]}'
)