From c084f10fe233ca0bc5d7ae5d1b342172ae6bfc53 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Mon, 21 Jul 2025 22:42:20 +0200 Subject: [PATCH 01/21] Add support for a database backup label instead of host:port (#1116) --- NEWS | 1 + borgmatic/config/schema.yaml | 25 +++++++++++++++++++++++ borgmatic/hooks/data_source/dump.py | 10 ++++----- borgmatic/hooks/data_source/mariadb.py | 7 ++++--- borgmatic/hooks/data_source/mongodb.py | 8 +++++--- borgmatic/hooks/data_source/mysql.py | 7 ++++--- borgmatic/hooks/data_source/postgresql.py | 8 +++++--- borgmatic/hooks/data_source/sqlite.py | 10 +++++---- docs/how-to/backup-your-databases.md | 2 ++ tests/unit/hooks/data_source/test_dump.py | 7 +++++++ 10 files changed, 64 insertions(+), 21 deletions(-) diff --git a/NEWS b/NEWS index 4c0e4f3e..639b9186 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 2.0.8.dev0 + * #1116: Add support for database backup labels. * #1114: Document systemd configuration changes for the ZFS filesystem hook. * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in use. diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0a49416a..5562c428 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1323,6 +1323,11 @@ properties: implicitly enables read_special (see above) to support dump and restore streaming. example: users + label: + type: string + description: | + Label to identify the database dump in the backup. + example: my_backup_label hostname: type: string description: | @@ -1524,6 +1529,11 @@ properties: database hook implicitly enables read_special (see above) to support dump and restore streaming. example: users + label: + type: string + description: | + Label to identify the database dump in the backup. + example: my_backup_label hostname: type: string description: | @@ -1689,6 +1699,11 @@ properties: database hook implicitly enables read_special (see above) to support dump and restore streaming. example: users + label: + type: string + description: | + Label to identify the database dump in the backup. + example: my_backup_label hostname: type: string description: | @@ -1862,6 +1877,11 @@ properties: read_special (see above) to support dump and restore streaming. example: /var/lib/sqlite/users.db + label: + type: string + description: | + Label to identify the database dump in the backup. + example: my_backup_label restore_path: type: string description: | @@ -1910,6 +1930,11 @@ properties: database hook implicitly enables read_special (see above) to support dump and restore streaming. example: users + label: + type: string + description: | + Label to identify the database dump in the backup. + example: my_backup_label hostname: type: string description: | diff --git a/borgmatic/hooks/data_source/dump.py b/borgmatic/hooks/data_source/dump.py index 3b746f7f..af64e0fc 100644 --- a/borgmatic/hooks/data_source/dump.py +++ b/borgmatic/hooks/data_source/dump.py @@ -19,7 +19,7 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam return os.path.join(borgmatic_runtime_directory, data_source_hook_name) -def make_data_source_dump_filename(dump_path, name, hostname=None, port=None): +def make_data_source_dump_filename(dump_path, name, hostname=None, port=None, label=None): ''' Based on the given dump directory path, data source name, hostname, and port, return a filename to use for the data source dump. The hostname defaults to localhost. @@ -29,12 +29,12 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None): if os.path.sep in name: raise ValueError(f'Invalid data source name {name}') - return os.path.join( - dump_path, - (hostname or 'localhost') + ('' if port is None else f':{port}'), - name, + identifier = ( + label if label else (hostname or 'localhost') + ('' if port is None else f':{port}') ) + return os.path.join(dump_path, identifier, name) + def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata): ''' diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 50d66ea6..4bd10bab 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -178,6 +178,7 @@ def execute_dump_command( database['name'], database.get('hostname'), database.get('port'), + database.get('label'), ) if os.path.exists(dump_filename): @@ -384,16 +385,16 @@ def make_data_source_dump_patterns( borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) return ( - dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'), + dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'), dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, - hostname='*', + label='*', ), dump.make_data_source_dump_filename( make_dump_path(borgmatic_source_directory), name, - hostname='*', + label='*', ), ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 3a249aaa..528ad607 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -71,6 +71,7 @@ def dump_data_sources( name, database.get('hostname'), database.get('port'), + database.get('label'), ) dump_format = database.get('format', 'archive') @@ -200,16 +201,16 @@ def make_data_source_dump_patterns( borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) return ( - dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'), + dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'), dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, - hostname='*', + label='*', ), dump.make_data_source_dump_filename( make_dump_path(borgmatic_source_directory), name, - hostname='*', + label='*', ), ) @@ -238,6 +239,7 @@ def restore_data_source_dump( make_dump_path(borgmatic_runtime_directory), data_source['name'], data_source.get('hostname'), + data_source.get('label'), ) restore_command = build_restore_command( extract_process, diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index ea4aae29..48991902 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -104,6 +104,7 @@ def execute_dump_command( database['name'], database.get('hostname'), database.get('port'), + database.get('label'), ) if os.path.exists(dump_filename): @@ -315,16 +316,16 @@ def make_data_source_dump_patterns( borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) return ( - dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'), + dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'), dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, - hostname='*', + label='*', ), dump.make_data_source_dump_filename( make_dump_path(borgmatic_source_directory), name, - hostname='*', + label='*', ), ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index f7eac3a2..695d79d9 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -186,6 +186,7 @@ def dump_data_sources( database_name, database.get('hostname'), database.get('port'), + database.get('label'), ) if os.path.exists(dump_filename): @@ -302,16 +303,16 @@ def make_data_source_dump_patterns( borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) return ( - dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'), + dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'), dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, - hostname='*', + label='*', ), dump.make_data_source_dump_filename( make_dump_path(borgmatic_source_directory), name, - hostname='*', + label='*', ), ) @@ -359,6 +360,7 @@ def restore_data_source_dump( make_dump_path(borgmatic_runtime_directory), data_source['name'], data_source.get('hostname'), + data_source.get('label'), ) psql_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql') diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 28555e5b..72fe059b 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -71,7 +71,9 @@ def dump_data_sources( ) dump_path = make_dump_path(borgmatic_runtime_directory) - dump_filename = dump.make_data_source_dump_filename(dump_path, database['name']) + dump_filename = dump.make_data_source_dump_filename( + dump_path, database['name'], label=database.get('label') + ) if os.path.exists(dump_filename): logger.warning( @@ -143,16 +145,16 @@ def make_data_source_dump_patterns( borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) return ( - dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'), + dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'), dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, - hostname='*', + label='*', ), dump.make_data_source_dump_filename( make_dump_path(borgmatic_source_directory), name, - hostname='*', + label='*', ), ) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 1239c1ec..4b3dc742 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -72,8 +72,10 @@ Here's a more involved example that connects to remote databases: ```yaml postgresql_databases: - name: users + label: database_server1 hostname: database1.example.org - name: orders + label: database_server2 hostname: database2.example.org port: 5433 username: postgres diff --git a/tests/unit/hooks/data_source/test_dump.py b/tests/unit/hooks/data_source/test_dump.py index 9aa35884..0acc3f5d 100644 --- a/tests/unit/hooks/data_source/test_dump.py +++ b/tests/unit/hooks/data_source/test_dump.py @@ -25,6 +25,13 @@ def test_make_data_source_dump_filename_uses_name_and_hostname_and_port(): ) +def test_make_data_source_dump_filename_users_label(): + assert ( + module.make_data_source_dump_filename('databases', 'test', 'hostname', 1234, 'custom_label') + == 'databases/custom_label/test' + ) + + def test_make_data_source_dump_filename_without_hostname_defaults_to_localhost(): assert module.make_data_source_dump_filename('databases', 'test') == 'databases/localhost/test' From 133a11c647c5ad482d7b29c81ce18d5ec0fb56bb Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 24 Jul 2025 08:51:39 +0200 Subject: [PATCH 02/21] add support for container names/id when dumping databases. --- NEWS | 1 + borgmatic/config/schema.yaml | 24 ++++++++++++ borgmatic/hooks/data_source/mariadb.py | 20 +++++----- borgmatic/hooks/data_source/mongodb.py | 18 +++++---- borgmatic/hooks/data_source/mysql.py | 20 +++++----- borgmatic/hooks/data_source/postgresql.py | 21 +++++----- borgmatic/hooks/data_source/utils.py | 47 +++++++++++++++++++++++ docs/how-to/backup-your-databases.md | 25 +++--------- tests/unit/hooks/data_source/test_dump.py | 6 ++- 9 files changed, 125 insertions(+), 57 deletions(-) create mode 100644 borgmatic/hooks/data_source/utils.py diff --git a/NEWS b/NEWS index 639b9186..762cebd3 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 2.0.8.dev0 + * #1116: Add support for dumping database containers via their container name. * #1116: Add support for database backup labels. * #1114: Document systemd configuration changes for the ZFS filesystem hook. * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 5562c428..f53482c6 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1328,6 +1328,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | @@ -1534,6 +1540,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | @@ -1704,6 +1716,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | @@ -1935,6 +1953,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 4bd10bab..8d26d23e 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -12,7 +12,7 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -122,6 +122,7 @@ def database_names_to_dump(database, config, username, password, environment, dr ) extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) show_command = ( mariadb_show_command + ( @@ -130,9 +131,9 @@ def database_names_to_dump(database, config, username, password, environment, dr else () ) + extra_options - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -176,9 +177,9 @@ def execute_dump_command( dump_filename = dump.make_data_source_dump_filename( dump_path, database['name'], - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) if os.path.exists(dump_filename): @@ -193,6 +194,7 @@ def execute_dump_command( ) extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) dump_command = ( mariadb_dump_command + ( @@ -202,9 +204,9 @@ def execute_dump_command( ) + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -417,7 +419,7 @@ def restore_data_source_dump( dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( 'restore_hostname', - data_source.get('hostname'), + utils.get_hostname_from_config(data_source), ) port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 528ad607..c940ca10 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -6,7 +6,7 @@ import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.hooks.credential.parse from borgmatic.execute import execute_command, execute_command_with_processes -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -69,9 +69,9 @@ def dump_data_sources( dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) dump_format = database.get('format', 'archive') @@ -139,10 +139,11 @@ def build_dump_command(database, config, dump_filename, dump_format): dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump') ) + hostname = utils.get_hostname_from_config(database) return ( dump_command + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) - + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + + (('--host', shlex.quote(hostname)) if hostname else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + ( ( @@ -238,8 +239,9 @@ def restore_data_source_dump( dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), data_source['name'], - data_source.get('hostname'), - data_source.get('label'), + hostname=data_source.get('hostname'), + port=data_source.get('port'), + label=data_source.get('label', data_source.get('container')), ) restore_command = build_restore_command( extract_process, @@ -269,7 +271,7 @@ def build_restore_command(extract_process, database, config, dump_filename, conn ''' hostname = connection_params['hostname'] or database.get( 'restore_hostname', - database.get('hostname'), + utils.get_hostname_from_config(database), ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) username = borgmatic.hooks.credential.parse.resolve_credential( diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 48991902..34e18ea2 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -12,7 +12,7 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -47,6 +47,7 @@ def database_names_to_dump(database, config, username, password, environment, dr borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) ) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) show_command = ( mysql_show_command + ( @@ -59,9 +60,9 @@ def database_names_to_dump(database, config, username, password, environment, dr else () ) + extra_options - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -102,9 +103,9 @@ def execute_dump_command( dump_filename = dump.make_data_source_dump_filename( dump_path, database['name'], - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) if os.path.exists(dump_filename): @@ -120,6 +121,7 @@ def execute_dump_command( borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) ) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) dump_command = ( mysql_dump_command + ( @@ -133,9 +135,9 @@ def execute_dump_command( ) + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -348,7 +350,7 @@ def restore_data_source_dump( dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( 'restore_hostname', - data_source.get('hostname'), + utils.get_hostname_from_config(data_source), ) port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 695d79d9..f3e41dc4 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -13,7 +13,7 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -91,10 +91,11 @@ def database_names_to_dump(database, config, environment, dry_run): psql_command = tuple( shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql') ) + hostname = utils.get_hostname_from_config(database) list_command = ( psql_command + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + ( ( @@ -184,9 +185,9 @@ def dump_data_sources( dump_filename = dump.make_data_source_dump_filename( dump_path, database_name, - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) if os.path.exists(dump_filename): @@ -195,6 +196,7 @@ def dump_data_sources( ) continue + hostname = utils.get_hostname_from_config(database) command = ( dump_command + ( @@ -202,7 +204,7 @@ def dump_data_sources( '--clean', '--if-exists', ) - + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + + (('--host', shlex.quote(hostname)) if hostname else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + ( ( @@ -342,7 +344,7 @@ def restore_data_source_dump( dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( 'restore_hostname', - data_source.get('hostname'), + utils.get_hostname_from_config(data_source), ) port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), @@ -359,8 +361,9 @@ def restore_data_source_dump( dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), data_source['name'], - data_source.get('hostname'), - data_source.get('label'), + hostname=data_source.get('hostname'), + port=data_source.get('port'), + label=data_source.get('label', data_source.get('container')), ) psql_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql') diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/utils.py new file mode 100644 index 00000000..5709c228 --- /dev/null +++ b/borgmatic/hooks/data_source/utils.py @@ -0,0 +1,47 @@ +import json +import shutil +import subprocess + +from borgmatic.execute import execute_command_and_capture_output + +IS_A_HOOK = False + + +def get_hostname_from_config(database): + if 'container' in database: + return get_ip_from_container(database['container']) + return database.get('hostname', '') + + +def get_ip_from_container(container): + engines = (shutil.which(engine) for engine in ('docker', 'podman')) + engines = [engine for engine in engines if engine] + + if not engines: + raise xxx # TODO: What to raise here, tell the user to install docker/podman + + for engine in engines: + try: + output = execute_command_and_capture_output( + ( + engine, + 'container', + 'inspect', + '--format={{json .NetworkSettings}}', + container, + ) + ) + except subprocess.CalledProcessError: + continue # Container does not exist + + network_data = json.loads(output.strip()) + main_ip = network_data.get('IPAddress') + if main_ip: + return main_ip + # No main IP found, try the networks + for network in network_data.get('Networks', {}).values(): + ip = network.get('IPAddress') + if ip: + return ip + + raise xxx # No container ip found, what to raise here diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 4b3dc742..dc67691e 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -214,35 +214,20 @@ these options in the `hooks:` section of your configuration. #### Database client on the host -But what if borgmatic is running on the host? You can still connect to a -database server container if its ports are properly exposed to the host. For -instance, when running the database container, you can specify `--publish -127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433 -on the host (only reachable on localhost, in this case). Or the same thing with -Docker Compose: - -```yaml -services: - your-database-server-container-name: - image: postgres - ports: - - 127.0.0.1:5433:5432 -``` - -And then you can configure borgmatic running on the host to connect to the -database: +But what if borgmatic is running on the host? You can connect to the database +container by specifying its container name or id: ```yaml postgresql_databases: - name: users - hostname: 127.0.0.1 + container: your-database-server-container-name port: 5433 username: postgres password: trustsome1 ``` -Alter the ports in these examples to suit your particular database system. - +Now borgmatic will use the `docker`/`podman` CLI to figure out the container IP. +Alternatively you can publish your container ports to the host. #### Database client in a running container diff --git a/tests/unit/hooks/data_source/test_dump.py b/tests/unit/hooks/data_source/test_dump.py index 0acc3f5d..a18fae3c 100644 --- a/tests/unit/hooks/data_source/test_dump.py +++ b/tests/unit/hooks/data_source/test_dump.py @@ -25,9 +25,11 @@ def test_make_data_source_dump_filename_uses_name_and_hostname_and_port(): ) -def test_make_data_source_dump_filename_users_label(): +def test_make_data_source_dump_filename_uses_label(): assert ( - module.make_data_source_dump_filename('databases', 'test', 'hostname', 1234, 'custom_label') + module.make_data_source_dump_filename( + 'databases', 'test', 'hostname', 1234, label='custom_label' + ) == 'databases/custom_label/test' ) From e058df6b7ed08306af96f2b21cf0b0d787bf8cee Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 26 Jul 2025 21:22:32 +0200 Subject: [PATCH 03/21] cleanup database option resolving. --- borgmatic/commands/arguments.py | 4 ++ borgmatic/hooks/data_source/mariadb.py | 25 ++++-------- borgmatic/hooks/data_source/mongodb.py | 26 +++++-------- borgmatic/hooks/data_source/mysql.py | 25 ++++-------- borgmatic/hooks/data_source/postgresql.py | 47 ++++++++--------------- borgmatic/hooks/data_source/utils.py | 32 +++++++++++++-- 6 files changed, 75 insertions(+), 84 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e35cf5a5..cfa5da2c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1475,6 +1475,10 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 '--port', help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration', ) + restore_group.add_argument( + '--container', + help='Container to restore to. Defaults to the "restore_container" option in borgmatic\'s configuration', + ) restore_group.add_argument( '--username', help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration', diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 8d26d23e..95949c1d 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -122,7 +122,7 @@ def database_names_to_dump(database, config, username, password, environment, dr ) extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) show_command = ( mariadb_show_command + ( @@ -194,7 +194,7 @@ def execute_dump_command( ) extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) dump_command = ( mariadb_dump_command + ( @@ -417,26 +417,17 @@ def restore_data_source_dump( subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = connection_params['hostname'] or data_source.get( - 'restore_hostname', - utils.get_hostname_from_config(data_source), + hostname = utils.resolve_database_option( + 'hostname', data_source, connection_params, restore=True ) - port = str( - connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), - ) - tls = data_source.get('restore_tls', data_source.get('tls')) + port = utils.resolve_database_option('port', data_source, connection_params, restore=True) + tls = utils.resolve_database_option('tls', data_source, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) - ), + utils.resolve_database_option('username', data_source, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['password'] - or data_source.get('restore_password', data_source.get('password')) - ), + utils.resolve_database_option('password', data_source, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index c940ca10..a2d7d90f 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -139,7 +139,7 @@ def build_dump_command(database, config, dump_filename, dump_format): dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump') ) - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) return ( dump_command + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) @@ -160,7 +160,10 @@ def build_dump_command(database, config, dump_filename, dump_format): ) + (('--config', make_password_config_file(password)) if password else ()) + ( - ('--authenticationDatabase', shlex.quote(database['authentication_database'])) + ( + '--authenticationDatabase', + shlex.quote(database['authentication_database']), + ) if 'authentication_database' in database else () ) @@ -251,7 +254,7 @@ def restore_data_source_dump( connection_params, ) - logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}") + logger.debug(f'Restoring MongoDB database {data_source["name"]}{dry_run_label}') if dry_run: return @@ -269,23 +272,14 @@ def build_restore_command(extract_process, database, config, dump_filename, conn ''' Return the custom mongorestore_command from a single database configuration. ''' - hostname = connection_params['hostname'] or database.get( - 'restore_hostname', - utils.get_hostname_from_config(database), - ) - port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) + hostname = utils.resolve_database_option('hostname', database, connection_params, restore=True) + port = utils.resolve_database_option('port', database, connection_params, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or database.get('restore_username', database.get('username')) - ), + utils.resolve_database_option('username', database, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['password'] - or database.get('restore_password', database.get('password')) - ), + utils.resolve_database_option('password', database, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 34e18ea2..9553b0ed 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -47,7 +47,7 @@ def database_names_to_dump(database, config, username, password, environment, dr borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) ) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) show_command = ( mysql_show_command + ( @@ -121,7 +121,7 @@ def execute_dump_command( borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) ) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) dump_command = ( mysql_dump_command + ( @@ -348,26 +348,17 @@ def restore_data_source_dump( subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = connection_params['hostname'] or data_source.get( - 'restore_hostname', - utils.get_hostname_from_config(data_source), + hostname = utils.resolve_database_option( + 'hostname', data_source, connection_params, restore=True ) - port = str( - connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), - ) - tls = data_source.get('restore_tls', data_source.get('tls')) + port = utils.resolve_database_option('port', data_source, connection_params, restore=True) + tls = utils.resolve_database_option('tls', data_source, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) - ), + utils.resolve_database_option('username', data_source, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['password'] - or data_source.get('restore_password', data_source.get('password')) - ), + utils.resolve_database_option('password', data_source, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index f3e41dc4..126428b4 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -32,22 +32,15 @@ def make_environment(database, config, restore_connection_params=None): ''' environment = dict(os.environ) - try: - if restore_connection_params: - environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( - ( - restore_connection_params.get('password') - or database.get('restore_password', database['password']) - ), - config, - ) - else: - environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( - database['password'], - config, - ) - except (AttributeError, KeyError): - pass + password = utils.resolve_database_option( + 'password', database, restore_connection_params, restore=restore_connection_params + ) + + if password: + environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( + password, + config, + ) if 'ssl_mode' in database: environment['PGSSLMODE'] = database['ssl_mode'] @@ -91,7 +84,7 @@ def database_names_to_dump(database, config, environment, dry_run): psql_command = tuple( shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql') ) - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) list_command = ( psql_command + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') @@ -196,7 +189,7 @@ def dump_data_sources( ) continue - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) command = ( dump_command + ( @@ -342,18 +335,12 @@ def restore_data_source_dump( hostname, port, username, and password. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = connection_params['hostname'] or data_source.get( - 'restore_hostname', - utils.get_hostname_from_config(data_source), - ) - port = str( - connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), + hostname = utils.resolve_database_option( + 'hostname', data_source, connection_params, restore=True ) + port = utils.resolve_database_option('port', data_source, connection_params, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) - ), + utils.resolve_database_option('username', data_source, connection_params, restore=True), config, ) @@ -372,7 +359,7 @@ def restore_data_source_dump( psql_command + ('--no-password', '--no-psqlrc', '--quiet') + (('--host', hostname) if hostname else ()) - + (('--port', port) if port else ()) + + (('--port', str(port)) if port else ()) + (('--username', username) if username else ()) + (('--dbname', data_source['name']) if not all_databases else ()) + ( @@ -393,7 +380,7 @@ def restore_data_source_dump( + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--dbname', data_source['name']) if not all_databases else ()) + (('--host', hostname) if hostname else ()) - + (('--port', port) if port else ()) + + (('--port', str(port)) if port else ()) + (('--username', username) if username else ()) + (('--no-owner',) if data_source.get('no_owner', False) else ()) + ( diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/utils.py index 5709c228..1cd56312 100644 --- a/borgmatic/hooks/data_source/utils.py +++ b/borgmatic/hooks/data_source/utils.py @@ -7,10 +7,34 @@ from borgmatic.execute import execute_command_and_capture_output IS_A_HOOK = False -def get_hostname_from_config(database): - if 'container' in database: - return get_ip_from_container(database['container']) - return database.get('hostname', '') +def resolve_database_option(option, data_source, connection_params=None, restore=False): + # Special case `hostname` since it overlaps with `container` + if option == 'hostname': + return _get_hostname_from_config(data_source, connection_params, restore) + if connection_params and (value := connection_params.get(option)): + return value + if restore and f'restore_{option}' in data_source: + return data_source[f'restore_{option}'] + return data_source.get(option) + + +def _get_hostname_from_config(data_source, connection_params=None, restore=False): + # connection params win, full stop + if connection_params: + if container := connection_params.get('container'): + return container + if hostname := connection_params.get('hostname'): + return hostname + # ... then try the restore config + if restore: + if 'restore_container' in data_source: + return get_ip_from_container(data_source['restore_container']) + if 'restore_hostname' in data_source: + return data_source['restore_hostname'] + # ... and finally fall back to the normal options + if 'container' in data_source: + return get_ip_from_container(data_source['container']) + return data_source.get('hostname', '') def get_ip_from_container(container): From bcacbc0d695c270f2f202cd57fcf5114e8be1390 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 26 Jul 2025 21:25:34 +0200 Subject: [PATCH 04/21] restore_container schema --- borgmatic/config/schema.yaml | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f53482c6..b01871da 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1330,10 +1330,16 @@ properties: example: my_backup_label container: type: string - description: + description: | Container name/id to connect to. When specified the hostname is ignored. Requires docker/podman CLI. example: debian_stable + restore_container: + type: string + description: | + Container name/id to restore to. Defaults to the "container" + option. + example: restore_container hostname: type: string description: | @@ -1542,10 +1548,16 @@ properties: example: my_backup_label container: type: string - description: + description: | Container name/id to connect to. When specified the hostname is ignored. Requires docker/podman CLI. example: debian_stable + restore_container: + type: string + description: | + Container name/id to restore to. Defaults to the "container" + option. + example: restore_container hostname: type: string description: | @@ -1718,10 +1730,16 @@ properties: example: my_backup_label container: type: string - description: + description: | Container name/id to connect to. When specified the hostname is ignored. Requires docker/podman CLI. example: debian_stable + restore_container: + type: string + description: | + Container name/id to restore to. Defaults to the "container" + option. + example: restore_container hostname: type: string description: | @@ -1955,10 +1973,16 @@ properties: example: my_backup_label container: type: string - description: + description: | Container name/id to connect to. When specified the hostname is ignored. Requires docker/podman CLI. example: debian_stable + restore_container: + type: string + description: | + Container name/id to restore to. Defaults to the "container" + option. + example: restore_container hostname: type: string description: | From db1c6f548f7ea4da13a191bcf65246252f18ab08 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 26 Jul 2025 22:01:58 +0200 Subject: [PATCH 05/21] add support for original-label and fix the tests from the previous changes. --- borgmatic/actions/restore.py | 20 +++++++++----- borgmatic/commands/arguments.py | 4 +++ borgmatic/config/schema.yaml | 16 +++++------ borgmatic/hooks/data_source/utils.py | 14 +++++++--- tests/unit/actions/test_restore.py | 40 +++++++++++++++++++++------- 5 files changed, 67 insertions(+), 27 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index fe68d256..cf5f6eef 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -22,8 +22,8 @@ UNSPECIFIED = object() Dump = collections.namedtuple( 'Dump', - ('hook_name', 'data_source_name', 'hostname', 'port'), - defaults=('localhost', None), + ('hook_name', 'data_source_name', 'hostname', 'port', 'label'), + defaults=('localhost', None, None), ) @@ -57,11 +57,14 @@ def render_dump_metadata(dump): ''' Given a Dump instance, make a display string describing it for use in log messages. ''' + label = dump.label or UNSPECIFIED name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name hostname = dump.hostname or UNSPECIFIED port = None if dump.port is UNSPECIFIED else dump.port - if port: + if label is not UNSPECIFIED: + metadata = f'{name}@{label}' + elif port: metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}' else: metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}' @@ -101,6 +104,7 @@ def get_configured_data_source(config, restore_dump): hook_data_source.get('name'), hook_data_source.get('hostname', 'localhost'), hook_data_source.get('port'), + hook_data_source.get('label'), ), restore_dump, default_port, @@ -172,7 +176,7 @@ def restore_single_dump( that data source from the archive. ''' dump_metadata = render_dump_metadata( - Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port')), + Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'), data_source.get('label')), ) logger.info(f'Restoring data source {dump_metadata}') @@ -371,7 +375,7 @@ def collect_dumps_from_archive( except (ValueError, TypeError): port = None - dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port)) + dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port, host_and_port)) # We've successfully parsed the dump path, so need to probe any further. break @@ -408,6 +412,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive): data_source_name=name, hostname=restore_arguments.original_hostname or UNSPECIFIED, port=restore_arguments.original_port, + label=restore_arguments.original_label, ) for name in restore_arguments.data_sources or (UNSPECIFIED,) } @@ -415,12 +420,14 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive): or restore_arguments.data_sources or restore_arguments.original_hostname or restore_arguments.original_port + or restore_arguments.original_label else { Dump( hook_name=UNSPECIFIED, data_source_name='all', hostname=UNSPECIFIED, port=UNSPECIFIED, + label=UNSPECIFIED, ), } ) @@ -541,6 +548,7 @@ def run_restore( dumps_actually_restored = set() connection_params = { + 'container': restore_arguments.container, 'hostname': restore_arguments.hostname, 'port': restore_arguments.port, 'username': restore_arguments.username, @@ -560,7 +568,7 @@ def run_restore( if not found_data_source: found_data_source = get_configured_data_source( config, - Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port), + Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port, restore_dump.label), ) if not found_data_source: diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index cfa5da2c..9292300f 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1491,6 +1491,10 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 '--restore-path', help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration', ) + restore_group.add_argument( + '--original-label', + help='The label where to dump to restore came from, only necessary if you need to disambiguate dumps', + ) restore_group.add_argument( '--original-hostname', help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps', diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index b01871da..36da3d5e 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1337,8 +1337,8 @@ properties: restore_container: type: string description: | - Container name/id to restore to. Defaults to the "container" - option. + Container name/id to restore to. Defaults to the + "container" option. example: restore_container hostname: type: string @@ -1555,8 +1555,8 @@ properties: restore_container: type: string description: | - Container name/id to restore to. Defaults to the "container" - option. + Container name/id to restore to. Defaults to the + "container" option. example: restore_container hostname: type: string @@ -1737,8 +1737,8 @@ properties: restore_container: type: string description: | - Container name/id to restore to. Defaults to the "container" - option. + Container name/id to restore to. Defaults to the + "container" option. example: restore_container hostname: type: string @@ -1980,8 +1980,8 @@ properties: restore_container: type: string description: | - Container name/id to restore to. Defaults to the "container" - option. + Container name/id to restore to. Defaults to the + "container" option. example: restore_container hostname: type: string diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/utils.py index 1cd56312..a788ce8e 100644 --- a/borgmatic/hooks/data_source/utils.py +++ b/borgmatic/hooks/data_source/utils.py @@ -1,4 +1,5 @@ import json +import logging import shutil import subprocess @@ -6,6 +7,7 @@ from borgmatic.execute import execute_command_and_capture_output IS_A_HOOK = False +logger = logging.getLogger(__name__) def resolve_database_option(option, data_source, connection_params=None, restore=False): # Special case `hostname` since it overlaps with `container` @@ -42,8 +44,9 @@ def get_ip_from_container(container): engines = [engine for engine in engines if engine] if not engines: - raise xxx # TODO: What to raise here, tell the user to install docker/podman + raise ValueError("Neither 'docker' nor 'podman' could be found on the system") + last_error = None for engine in engines: try: output = execute_command_and_capture_output( @@ -55,7 +58,9 @@ def get_ip_from_container(container): container, ) ) - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as error: + last_error = error + logger.debug(f"Couldn't find container '{container}' with engine '{engine}'") continue # Container does not exist network_data = json.loads(output.strip()) @@ -68,4 +73,7 @@ def get_ip_from_container(container): if ip: return ip - raise xxx # No container ip found, what to raise here + if last_error: + raise last_error + + return None diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 9941fdb3..602c30b4 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -172,6 +172,10 @@ def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values( module.Dump('postgresql_databases', 'foo', 'host', module.UNSPECIFIED), 'foo@host (postgresql_databases)', ), + ( + module.Dump('postgresql_databases', 'foo', 'host', 1234, 'label'), + 'foo@label (postgresql_databases)', + ), ( module.Dump( module.UNSPECIFIED, @@ -580,9 +584,9 @@ def test_collect_dumps_from_archive_without_dumps_metadata_falls_back_to_parsing ) assert archive_dumps == { - module.Dump('postgresql_databases', 'foo'), - module.Dump('postgresql_databases', 'bar', 'host', 1234), - module.Dump('mysql_databases', 'quux'), + module.Dump('postgresql_databases', 'foo', label='localhost'), + module.Dump('postgresql_databases', 'bar', 'host', 1234, 'host:1234'), + module.Dump('mysql_databases', 'quux', label='localhost'), } @@ -623,10 +627,10 @@ def test_collect_dumps_from_archive_parses_archive_paths_with_different_base_dir ) assert archive_dumps == { - module.Dump('postgresql_databases', 'foo'), - module.Dump('postgresql_databases', 'bar'), - module.Dump('postgresql_databases', 'baz'), - module.Dump('mysql_databases', 'quux'), + module.Dump('postgresql_databases', 'foo', label='localhost'), + module.Dump('postgresql_databases', 'bar', label='localhost'), + module.Dump('postgresql_databases', 'baz', label='localhost'), + module.Dump('mysql_databases', 'quux', label='localhost'), } @@ -665,7 +669,7 @@ def test_collect_dumps_from_archive_parses_directory_format_archive_paths(): ) assert archive_dumps == { - module.Dump('postgresql_databases', 'foo'), + module.Dump('postgresql_databases', 'foo', label='localhost'), } @@ -707,8 +711,8 @@ def test_collect_dumps_from_archive_skips_bad_archive_paths_or_bad_path_componen ) assert archive_dumps == { - module.Dump('postgresql_databases', 'foo'), - module.Dump('postgresql_databases', 'bar'), + module.Dump('postgresql_databases', 'foo', label='localhost'), + module.Dump('postgresql_databases', 'bar', label='localhost:abcd'), } @@ -734,6 +738,7 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive(): data_sources=['foo', 'bar'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) == { @@ -756,6 +761,7 @@ def test_get_dumps_to_restore_raises_for_requested_dumps_missing_from_archive(): data_sources=['foo', 'bar'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) @@ -775,6 +781,7 @@ def test_get_dumps_to_restore_without_requested_dumps_finds_all_archive_dumps(): data_sources=[], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) @@ -804,6 +811,7 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_finds_all_archive_dump data_sources=['all'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) @@ -833,6 +841,7 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques data_sources=['all', 'foo', 'bar'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) @@ -859,6 +868,7 @@ def test_get_dumps_to_restore_raises_for_multiple_matching_dumps_in_archive(): data_sources=['foo'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive={ module.Dump('postgresql_databases', 'foo'), @@ -882,6 +892,7 @@ def test_get_dumps_to_restore_raises_for_all_in_requested_dumps_and_requested_du data_sources=['all', 'foo', 'bar'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive={module.Dump('postresql_databases', 'foo')}, ) @@ -905,6 +916,7 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar data_sources=['foo'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) == { @@ -930,6 +942,7 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f data_sources=['foo'], original_hostname=None, original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) == { @@ -955,6 +968,7 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc data_sources=['foo'], original_hostname='host', original_port=None, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) == { @@ -980,6 +994,7 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive data_sources=['foo'], original_hostname='host', original_port=1234, + original_label=None, ), dumps_from_archive=dumps_from_archive, ) == { @@ -1087,6 +1102,7 @@ def test_run_restore_restores_each_data_source(): username=None, password=None, restore_path=None, + container=None ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), @@ -1171,6 +1187,7 @@ def test_run_restore_restores_data_source_by_falling_back_to_all_name(): username=None, password=None, restore_path=None, + container=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), @@ -1252,6 +1269,7 @@ def test_run_restore_restores_data_source_configured_with_all_name(): username=None, password=None, restore_path=None, + container=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), @@ -1333,6 +1351,7 @@ def test_run_restore_skips_missing_data_source(): username=None, password=None, restore_path=None, + container=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), @@ -1410,6 +1429,7 @@ def test_run_restore_restores_data_sources_from_different_hooks(): username=None, password=None, restore_path=None, + container=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), From 04cc58a5c79c476be1079ad636b2ea50d5e10a83 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 26 Jul 2025 22:41:14 +0200 Subject: [PATCH 06/21] fixes and more tests. --- borgmatic/actions/restore.py | 16 +++- borgmatic/hooks/data_source/utils.py | 3 +- tests/unit/actions/test_restore.py | 2 +- tests/unit/hooks/data_source/test_utils.py | 91 ++++++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 tests/unit/hooks/data_source/test_utils.py diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index cf5f6eef..632c0183 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -176,7 +176,13 @@ def restore_single_dump( that data source from the archive. ''' dump_metadata = render_dump_metadata( - Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'), data_source.get('label')), + Dump( + hook_name, + data_source['name'], + data_source.get('hostname'), + data_source.get('port'), + data_source.get('label'), + ), ) logger.info(f'Restoring data source {dump_metadata}') @@ -568,7 +574,13 @@ def run_restore( if not found_data_source: found_data_source = get_configured_data_source( config, - Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port, restore_dump.label), + Dump( + restore_dump.hook_name, + 'all', + restore_dump.hostname, + restore_dump.port, + restore_dump.label, + ), ) if not found_data_source: diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/utils.py index a788ce8e..6c50472d 100644 --- a/borgmatic/hooks/data_source/utils.py +++ b/borgmatic/hooks/data_source/utils.py @@ -9,6 +9,7 @@ IS_A_HOOK = False logger = logging.getLogger(__name__) + def resolve_database_option(option, data_source, connection_params=None, restore=False): # Special case `hostname` since it overlaps with `container` if option == 'hostname': @@ -24,7 +25,7 @@ def _get_hostname_from_config(data_source, connection_params=None, restore=False # connection params win, full stop if connection_params: if container := connection_params.get('container'): - return container + return get_ip_from_container(container) if hostname := connection_params.get('hostname'): return hostname # ... then try the restore config diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 602c30b4..106d1e01 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -1102,7 +1102,7 @@ def test_run_restore_restores_each_data_source(): username=None, password=None, restore_path=None, - container=None + container=None, ), global_arguments=flexmock(dry_run=False), local_path=flexmock(), diff --git a/tests/unit/hooks/data_source/test_utils.py b/tests/unit/hooks/data_source/test_utils.py new file mode 100644 index 00000000..ef7c0b2c --- /dev/null +++ b/tests/unit/hooks/data_source/test_utils.py @@ -0,0 +1,91 @@ +from subprocess import CalledProcessError + +import pytest +from flexmock import flexmock + +from borgmatic.hooks.data_source import utils + + +def test_get_database_option(): + data_source = {'option': 'original_value', 'restore_option': 'restore_value'} + connection_params = {'option': 'connection_value'} + + assert utils.resolve_database_option('option', data_source) == 'original_value' + assert utils.resolve_database_option('option', data_source, restore=True) == 'restore_value' + assert ( + utils.resolve_database_option('option', data_source, connection_params) + == 'connection_value' + ) + assert ( + utils.resolve_database_option('option', data_source, connection_params, restore=True) + == 'connection_value' + ) + + +def test_get_hostname_option_via_container(): + data_source = { + 'container': 'original_container', + 'hostname': 'original_hostname', + 'restore_container': 'restore_container', + 'restore_hostname': 'restore_hostname', + } + connection_params = {'container': 'connection_container', 'hostname': 'connection_hostname'} + + flexmock(utils).should_receive('get_ip_from_container').with_args( + 'original_container' + ).and_return('container_ip_1') + flexmock(utils).should_receive('get_ip_from_container').with_args( + 'connection_container' + ).and_return('container_ip_2') + flexmock(utils).should_receive('get_ip_from_container').with_args( + 'restore_container' + ).and_return('container_ip_3') + + assert utils.resolve_database_option('hostname', data_source) == 'container_ip_1' + assert ( + utils.resolve_database_option('hostname', data_source, connection_params) + == 'container_ip_2' + ) + assert utils.resolve_database_option('hostname', data_source, restore=True) == 'container_ip_3' + + +def test_get_container_ip_without_engines(): + flexmock(utils.shutil).should_receive('which').and_return(None).and_return(None) + + with pytest.raises(ValueError): + utils.get_ip_from_container('yolo') + + +def test_get_container_ip_success(): + flexmock(utils.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman') + + flexmock(utils).should_receive('execute_command_and_capture_output').and_return( + '{"IPAddress": "1.2.3.4"}' + ) + + addr = utils.get_ip_from_container('yolo') + assert addr == '1.2.3.4' + + flexmock(utils).should_receive('execute_command_and_capture_output').and_return( + '{"Networks": {"my_network": {"IPAddress": "5.6.7.8"}}}' + ) + + assert utils.get_ip_from_container('yolo') == '5.6.7.8' + + +def test_get_container_ip_container_not_found(): + flexmock(utils.shutil).should_receive('which').and_return('/usr/bin/podman') + flexmock(utils).should_receive('execute_command_and_capture_output').and_raise( + CalledProcessError, 1, ['/usr/bin/podman', 'inspect', 'yolo'], None, 'No such object' + ) + + with pytest.raises(CalledProcessError): + utils.get_ip_from_container('does not exist') + + +def test_get_container_ip_container_no_network(): + flexmock(utils.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman') + + flexmock(utils).should_receive('execute_command_and_capture_output').and_return('{}') + + assert utils.get_ip_from_container('yolo') is None From a2c9bb12e5736a77b099ce17a98efac23fc27f79 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sun, 27 Jul 2025 14:52:30 +0200 Subject: [PATCH 07/21] test fixes --- borgmatic/actions/restore.py | 5 ++--- borgmatic/hooks/data_source/utils.py | 6 ++++-- tests/unit/hooks/data_source/test_utils.py | 4 +++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 632c0183..fd6e435c 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -104,7 +104,7 @@ def get_configured_data_source(config, restore_dump): hook_data_source.get('name'), hook_data_source.get('hostname', 'localhost'), hook_data_source.get('port'), - hook_data_source.get('label'), + hook_data_source.get('label') or hook_data_source.get('container') or UNSPECIFIED, ), restore_dump, default_port, @@ -181,7 +181,7 @@ def restore_single_dump( data_source['name'], data_source.get('hostname'), data_source.get('port'), - data_source.get('label'), + data_source.get('label') or data_source.get('container') or UNSPECIFIED, ), ) @@ -568,7 +568,6 @@ def run_restore( config, restore_dump, ) - # For a dump that wasn't found via an exact match in the configuration, try to fallback # to an "all" data source. if not found_data_source: diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/utils.py index 6c50472d..194ec1be 100644 --- a/borgmatic/hooks/data_source/utils.py +++ b/borgmatic/hooks/data_source/utils.py @@ -61,7 +61,7 @@ def get_ip_from_container(container): ) except subprocess.CalledProcessError as error: last_error = error - logger.debug(f"Couldn't find container '{container}' with engine '{engine}'") + logger.debug(f"Could not find container '{container}' with engine '{engine}'") continue # Container does not exist network_data = json.loads(output.strip()) @@ -77,4 +77,6 @@ def get_ip_from_container(container): if last_error: raise last_error - return None + raise ValueError( + f"Could not determine ip address for container '{container}'; running in host mode or userspace networking?" + ) diff --git a/tests/unit/hooks/data_source/test_utils.py b/tests/unit/hooks/data_source/test_utils.py index ef7c0b2c..0db4c7ec 100644 --- a/tests/unit/hooks/data_source/test_utils.py +++ b/tests/unit/hooks/data_source/test_utils.py @@ -88,4 +88,6 @@ def test_get_container_ip_container_no_network(): flexmock(utils).should_receive('execute_command_and_capture_output').and_return('{}') - assert utils.get_ip_from_container('yolo') is None + with pytest.raises(ValueError) as exc_info: + utils.get_ip_from_container('yolo') + assert 'Could not determine ip address for container' in str(exc_info.value) From 3811a9f57b651afe923ccc470fb907a32b4c7b19 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sun, 27 Jul 2025 20:42:18 +0200 Subject: [PATCH 08/21] more test fixes and first stab at end-to-end tests --- borgmatic/actions/restore.py | 4 + tests/end-to-end/commands/docker | 2 + .../hooks/data_source/test_database.py | 118 ++++++++++++++---- tests/unit/actions/test_restore.py | 10 +- 4 files changed, 106 insertions(+), 28 deletions(-) create mode 100755 tests/end-to-end/commands/docker diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index fd6e435c..e348c387 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -33,6 +33,10 @@ def dumps_match(first, second, default_port=None): indicates that the field should match any value. If a default port is given, then consider any dump having that port to match with a dump having a None port. ''' + # label kinda counts as id, if they match, consider it a full match + if first.label not in {None, UNSPECIFIED} and first.label == second.label: + return True + for field_name in first._fields: first_value = getattr(first, field_name) second_value = getattr(second, field_name) diff --git a/tests/end-to-end/commands/docker b/tests/end-to-end/commands/docker new file mode 100755 index 00000000..971199a1 --- /dev/null +++ b/tests/end-to-end/commands/docker @@ -0,0 +1,2 @@ +#!/bin/sh +echo "{\"IPAddress\": \"$4\"}" diff --git a/tests/end-to-end/hooks/data_source/test_database.py b/tests/end-to-end/hooks/data_source/test_database.py index 90d0c264..fd9a800f 100644 --- a/tests/end-to-end/hooks/data_source/test_database.py +++ b/tests/end-to-end/hooks/data_source/test_database.py @@ -9,6 +9,8 @@ import pymongo import pytest import ruamel.yaml +from borgmatic.hooks.data_source import utils + def write_configuration( source_directory, @@ -19,6 +21,7 @@ def write_configuration( postgresql_all_dump_format=None, mariadb_mysql_all_dump_format=None, mongodb_dump_format='archive', + use_containers=False, ): ''' Write out borgmatic configuration into a file at the config path. Set the options so as to work @@ -32,6 +35,8 @@ def write_configuration( f'format: {mariadb_mysql_all_dump_format}' if mariadb_mysql_all_dump_format else '' ) + hostname_option = 'container' if use_containers else 'hostname' + config_yaml = f''' source_directories: - {source_directory} @@ -43,44 +48,44 @@ encryption_passphrase: "test" postgresql_databases: - name: test - hostname: postgresql + {hostname_option}: postgresql username: postgres password: test format: {postgresql_dump_format} - name: all {postgresql_all_format_option} - hostname: postgresql + {hostname_option}: postgresql username: postgres password: test mariadb_databases: - name: test - hostname: mariadb + {hostname_option}: mariadb username: root password: test - name: all {mariadb_mysql_dump_format_option} - hostname: mariadb + {hostname_option}: mariadb username: root password: test mysql_databases: - name: test - hostname: not-actually-mysql + {hostname_option}: not-actually-mysql username: root password: test - name: all {mariadb_mysql_dump_format_option} - hostname: not-actually-mysql + {hostname_option}: not-actually-mysql username: root password: test mongodb_databases: - name: test - hostname: mongodb + {hostname_option}: mongodb username: root password: test authentication_database: admin format: {mongodb_dump_format} - name: all - hostname: mongodb + {hostname_option}: mongodb username: root password: test sqlite_databases: @@ -110,12 +115,16 @@ def write_custom_restore_configuration( postgresql_all_dump_format=None, mariadb_mysql_all_dump_format=None, mongodb_dump_format='archive', + use_containers=False, ): ''' Write out borgmatic configuration into a file at the config path. Set the options so as to work for testing with custom restore options. This includes a custom restore_hostname, restore_port, restore_username, restore_password and restore_path. ''' + + hostname_option = 'container' if use_containers else 'hostname' + config_yaml = f''' source_directories: - {source_directory} @@ -127,39 +136,39 @@ encryption_passphrase: "test" postgresql_databases: - name: test - hostname: postgresql + {hostname_option}: postgresql username: postgres password: test format: {postgresql_dump_format} - restore_hostname: postgresql2 + restore_{hostname_option}: postgresql2 restore_port: 5433 restore_password: test2 mariadb_databases: - name: test - hostname: mariadb + {hostname_option}: mariadb username: root password: test - restore_hostname: mariadb2 + restore_{hostname_option}: mariadb2 restore_port: 3307 restore_username: root restore_password: test2 mysql_databases: - name: test - hostname: not-actually-mysql + {hostname_option}: not-actually-mysql username: root password: test - restore_hostname: not-actually-mysql2 + restore_{hostname_option}: not-actually-mysql2 restore_port: 3307 restore_username: root restore_password: test2 mongodb_databases: - name: test - hostname: mongodb + {hostname_option}: mongodb username: root password: test authentication_database: admin format: {mongodb_dump_format} - restore_hostname: mongodb2 + restore_{hostname_option}: mongodb2 restore_port: 27018 restore_username: root2 restore_password: test2 @@ -212,16 +221,10 @@ postgresql_databases: def get_connection_params(database, use_restore_options=False): - hostname = (database.get('restore_hostname') if use_restore_options else None) or database.get( - 'hostname', - ) - port = (database.get('restore_port') if use_restore_options else None) or database.get('port') - username = (database.get('restore_username') if use_restore_options else None) or database.get( - 'username', - ) - password = (database.get('restore_password') if use_restore_options else None) or database.get( - 'password', - ) + hostname = utils.resolve_database_option('hostname', database, restore=use_restore_options) + port = utils.resolve_database_option('port', database, restore=use_restore_options) + username = utils.resolve_database_option('username', database, restore=use_restore_options) + password = utils.resolve_database_option('password', database, restore=use_restore_options) return (hostname, port, username, password) @@ -656,3 +659,66 @@ def test_database_dump_with_error_causes_borgmatic_to_exit(): finally: os.chdir(original_working_directory) shutil.rmtree(temporary_directory) + + +def test_database_dump_and_restore_containers(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + original_path = os.environ.get('PATH', '') + + os.environ['PATH'] = f'/app/tests/end-to-end/commands:{original_path}' + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + config = write_configuration( + temporary_directory, + config_path, + repository_path, + temporary_directory, + use_containers=True, + ) + create_test_tables(config) + select_test_tables(config) + + subprocess.check_call( + [ + 'borgmatic', + '-v', + '2', + '--config', + config_path, + 'repo-create', + '--encryption', + 'repokey', + ], + ) + + # Run borgmatic to generate a backup archive including database dumps. + subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2']) + + # Get the created archive name. + output = subprocess.check_output( + ['borgmatic', '--config', config_path, 'list', '--json'], + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + archive_name = parsed_output[0]['archives'][0]['archive'] + + # Restore the databases from the archive. + drop_test_tables(config) + subprocess.check_call( + ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name], + ) + + # Ensure the test tables have actually been restored. + select_test_tables(config) + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) + drop_test_tables(config) + os.environ['PATH'] = original_path diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 106d1e01..ec8488b0 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -126,6 +126,12 @@ import borgmatic.actions.restore as module 5432, False, ), + ( + module.Dump('postgresql_databases', 'foo', 'some_host1', 5433, 'unique'), + module.Dump('postgresql_databases', 'foo', 'some_host2', None, 'unique'), + 5432, + True, + ), ), ) def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values( @@ -196,7 +202,7 @@ def test_get_configured_data_source_matches_data_source_with_restore_dump(): flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').and_return(default_port) flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump('postgresql_databases', 'bar'), + module.Dump('postgresql_databases', 'bar', label=module.UNSPECIFIED), module.Dump('postgresql_databases', 'bar'), default_port=default_port, ).and_return(True) @@ -243,7 +249,7 @@ def test_get_configured_data_source_with_multiple_matching_data_sources_errors() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').and_return(default_port) flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump('postgresql_databases', 'bar'), + module.Dump('postgresql_databases', 'bar', label=module.UNSPECIFIED), module.Dump('postgresql_databases', 'bar'), default_port=default_port, ).and_return(True) From 97815bb4980a070ea4cadf0367b89f22888df768 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sun, 27 Jul 2025 20:49:08 +0200 Subject: [PATCH 09/21] fix dump matching logic with labels. --- borgmatic/actions/restore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index e348c387..c325454b 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -33,9 +33,9 @@ def dumps_match(first, second, default_port=None): indicates that the field should match any value. If a default port is given, then consider any dump having that port to match with a dump having a None port. ''' - # label kinda counts as id, if they match, consider it a full match + # label kinda counts as id, if they match ignore hostname & port if first.label not in {None, UNSPECIFIED} and first.label == second.label: - return True + return first[:2] == second[:2] for field_name in first._fields: first_value = getattr(first, field_name) From 0d190016d31af50da0ba874c1a2fa3fa9358623f Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 31 Jul 2025 07:58:17 +0200 Subject: [PATCH 10/21] rename utils.py to config.py --- .../hooks/data_source/{utils.py => config.py} | 0 borgmatic/hooks/data_source/mariadb.py | 17 ++-- borgmatic/hooks/data_source/mongodb.py | 15 +-- borgmatic/hooks/data_source/mysql.py | 17 ++-- borgmatic/hooks/data_source/postgresql.py | 15 +-- .../hooks/data_source/test_database.py | 10 +- tests/unit/hooks/data_source/test_config.py | 93 +++++++++++++++++++ tests/unit/hooks/data_source/test_utils.py | 93 ------------------- 8 files changed, 133 insertions(+), 127 deletions(-) rename borgmatic/hooks/data_source/{utils.py => config.py} (100%) create mode 100644 tests/unit/hooks/data_source/test_config.py delete mode 100644 tests/unit/hooks/data_source/test_utils.py diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/config.py similarity index 100% rename from borgmatic/hooks/data_source/utils.py rename to borgmatic/hooks/data_source/config.py diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 95949c1d..774a0d4a 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -12,7 +12,8 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump, utils +from borgmatic.hooks.data_source import config as ds_config +from borgmatic.hooks.data_source import dump logger = logging.getLogger(__name__) @@ -122,7 +123,7 @@ def database_names_to_dump(database, config, username, password, environment, dr ) extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) password_transport = database.get('password_transport', 'pipe') - hostname = utils.resolve_database_option('hostname', database) + hostname = ds_config.resolve_database_option('hostname', database) show_command = ( mariadb_show_command + ( @@ -194,7 +195,7 @@ def execute_dump_command( ) extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) password_transport = database.get('password_transport', 'pipe') - hostname = utils.resolve_database_option('hostname', database) + hostname = ds_config.resolve_database_option('hostname', database) dump_command = ( mariadb_dump_command + ( @@ -417,17 +418,17 @@ def restore_data_source_dump( subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = utils.resolve_database_option( + hostname = ds_config.resolve_database_option( 'hostname', data_source, connection_params, restore=True ) - port = utils.resolve_database_option('port', data_source, connection_params, restore=True) - tls = utils.resolve_database_option('tls', data_source, restore=True) + port = ds_config.resolve_database_option('port', data_source, connection_params, restore=True) + tls = ds_config.resolve_database_option('tls', data_source, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - utils.resolve_database_option('username', data_source, connection_params, restore=True), + ds_config.resolve_database_option('username', data_source, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - utils.resolve_database_option('password', data_source, connection_params, restore=True), + ds_config.resolve_database_option('password', data_source, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index a2d7d90f..7b91f58d 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -6,7 +6,8 @@ import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.hooks.credential.parse from borgmatic.execute import execute_command, execute_command_with_processes -from borgmatic.hooks.data_source import dump, utils +from borgmatic.hooks.data_source import config as ds_config +from borgmatic.hooks.data_source import dump logger = logging.getLogger(__name__) @@ -139,7 +140,7 @@ def build_dump_command(database, config, dump_filename, dump_format): dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump') ) - hostname = utils.resolve_database_option('hostname', database) + hostname = ds_config.resolve_database_option('hostname', database) return ( dump_command + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) @@ -272,14 +273,16 @@ def build_restore_command(extract_process, database, config, dump_filename, conn ''' Return the custom mongorestore_command from a single database configuration. ''' - hostname = utils.resolve_database_option('hostname', database, connection_params, restore=True) - port = utils.resolve_database_option('port', database, connection_params, restore=True) + hostname = ds_config.resolve_database_option( + 'hostname', database, connection_params, restore=True + ) + port = ds_config.resolve_database_option('port', database, connection_params, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - utils.resolve_database_option('username', database, connection_params, restore=True), + ds_config.resolve_database_option('username', database, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - utils.resolve_database_option('password', database, connection_params, restore=True), + ds_config.resolve_database_option('password', database, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 9553b0ed..91ad4f8f 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -12,7 +12,8 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump, utils +from borgmatic.hooks.data_source import config as ds_config +from borgmatic.hooks.data_source import dump logger = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def database_names_to_dump(database, config, username, password, environment, dr borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) ) password_transport = database.get('password_transport', 'pipe') - hostname = utils.resolve_database_option('hostname', database) + hostname = ds_config.resolve_database_option('hostname', database) show_command = ( mysql_show_command + ( @@ -121,7 +122,7 @@ def execute_dump_command( borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) ) password_transport = database.get('password_transport', 'pipe') - hostname = utils.resolve_database_option('hostname', database) + hostname = ds_config.resolve_database_option('hostname', database) dump_command = ( mysql_dump_command + ( @@ -348,17 +349,17 @@ def restore_data_source_dump( subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = utils.resolve_database_option( + hostname = ds_config.resolve_database_option( 'hostname', data_source, connection_params, restore=True ) - port = utils.resolve_database_option('port', data_source, connection_params, restore=True) - tls = utils.resolve_database_option('tls', data_source, restore=True) + port = ds_config.resolve_database_option('port', data_source, connection_params, restore=True) + tls = ds_config.resolve_database_option('tls', data_source, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - utils.resolve_database_option('username', data_source, connection_params, restore=True), + ds_config.resolve_database_option('username', data_source, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - utils.resolve_database_option('password', data_source, connection_params, restore=True), + ds_config.resolve_database_option('password', data_source, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 126428b4..3da88345 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -13,7 +13,8 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump, utils +from borgmatic.hooks.data_source import config as ds_config +from borgmatic.hooks.data_source import dump logger = logging.getLogger(__name__) @@ -32,7 +33,7 @@ def make_environment(database, config, restore_connection_params=None): ''' environment = dict(os.environ) - password = utils.resolve_database_option( + password = ds_config.resolve_database_option( 'password', database, restore_connection_params, restore=restore_connection_params ) @@ -84,7 +85,7 @@ def database_names_to_dump(database, config, environment, dry_run): psql_command = tuple( shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql') ) - hostname = utils.resolve_database_option('hostname', database) + hostname = ds_config.resolve_database_option('hostname', database) list_command = ( psql_command + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') @@ -189,7 +190,7 @@ def dump_data_sources( ) continue - hostname = utils.resolve_database_option('hostname', database) + hostname = ds_config.resolve_database_option('hostname', database) command = ( dump_command + ( @@ -335,12 +336,12 @@ def restore_data_source_dump( hostname, port, username, and password. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = utils.resolve_database_option( + hostname = ds_config.resolve_database_option( 'hostname', data_source, connection_params, restore=True ) - port = utils.resolve_database_option('port', data_source, connection_params, restore=True) + port = ds_config.resolve_database_option('port', data_source, connection_params, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - utils.resolve_database_option('username', data_source, connection_params, restore=True), + ds_config.resolve_database_option('username', data_source, connection_params, restore=True), config, ) diff --git a/tests/end-to-end/hooks/data_source/test_database.py b/tests/end-to-end/hooks/data_source/test_database.py index fd9a800f..74030130 100644 --- a/tests/end-to-end/hooks/data_source/test_database.py +++ b/tests/end-to-end/hooks/data_source/test_database.py @@ -9,7 +9,7 @@ import pymongo import pytest import ruamel.yaml -from borgmatic.hooks.data_source import utils +from borgmatic.hooks.data_source import config def write_configuration( @@ -221,10 +221,10 @@ postgresql_databases: def get_connection_params(database, use_restore_options=False): - hostname = utils.resolve_database_option('hostname', database, restore=use_restore_options) - port = utils.resolve_database_option('port', database, restore=use_restore_options) - username = utils.resolve_database_option('username', database, restore=use_restore_options) - password = utils.resolve_database_option('password', database, restore=use_restore_options) + hostname = config.resolve_database_option('hostname', database, restore=use_restore_options) + port = config.resolve_database_option('port', database, restore=use_restore_options) + username = config.resolve_database_option('username', database, restore=use_restore_options) + password = config.resolve_database_option('password', database, restore=use_restore_options) return (hostname, port, username, password) diff --git a/tests/unit/hooks/data_source/test_config.py b/tests/unit/hooks/data_source/test_config.py new file mode 100644 index 00000000..ae3198d0 --- /dev/null +++ b/tests/unit/hooks/data_source/test_config.py @@ -0,0 +1,93 @@ +from subprocess import CalledProcessError + +import pytest +from flexmock import flexmock + +from borgmatic.hooks.data_source import config + + +def test_get_database_option(): + data_source = {'option': 'original_value', 'restore_option': 'restore_value'} + connection_params = {'option': 'connection_value'} + + assert config.resolve_database_option('option', data_source) == 'original_value' + assert config.resolve_database_option('option', data_source, restore=True) == 'restore_value' + assert ( + config.resolve_database_option('option', data_source, connection_params) + == 'connection_value' + ) + assert ( + config.resolve_database_option('option', data_source, connection_params, restore=True) + == 'connection_value' + ) + + +def test_get_hostname_option_via_container(): + data_source = { + 'container': 'original_container', + 'hostname': 'original_hostname', + 'restore_container': 'restore_container', + 'restore_hostname': 'restore_hostname', + } + connection_params = {'container': 'connection_container', 'hostname': 'connection_hostname'} + + flexmock(config).should_receive('get_ip_from_container').with_args( + 'original_container' + ).and_return('container_ip_1') + flexmock(config).should_receive('get_ip_from_container').with_args( + 'connection_container' + ).and_return('container_ip_2') + flexmock(config).should_receive('get_ip_from_container').with_args( + 'restore_container' + ).and_return('container_ip_3') + + assert config.resolve_database_option('hostname', data_source) == 'container_ip_1' + assert ( + config.resolve_database_option('hostname', data_source, connection_params) + == 'container_ip_2' + ) + assert config.resolve_database_option('hostname', data_source, restore=True) == 'container_ip_3' + + +def test_get_container_ip_without_engines(): + flexmock(config.shutil).should_receive('which').and_return(None).and_return(None) + + with pytest.raises(ValueError): + config.get_ip_from_container('yolo') + + +def test_get_container_ip_success(): + flexmock(config.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman') + + flexmock(config).should_receive('execute_command_and_capture_output').and_return( + '{"IPAddress": "1.2.3.4"}' + ) + + addr = config.get_ip_from_container('yolo') + assert addr == '1.2.3.4' + + flexmock(config).should_receive('execute_command_and_capture_output').and_return( + '{"Networks": {"my_network": {"IPAddress": "5.6.7.8"}}}' + ) + + assert config.get_ip_from_container('yolo') == '5.6.7.8' + + +def test_get_container_ip_container_not_found(): + flexmock(config.shutil).should_receive('which').and_return('/usr/bin/podman') + flexmock(config).should_receive('execute_command_and_capture_output').and_raise( + CalledProcessError, 1, ['/usr/bin/podman', 'inspect', 'yolo'], None, 'No such object' + ) + + with pytest.raises(CalledProcessError): + config.get_ip_from_container('does not exist') + + +def test_get_container_ip_container_no_network(): + flexmock(config.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman') + + flexmock(config).should_receive('execute_command_and_capture_output').and_return('{}') + + with pytest.raises(ValueError) as exc_info: + config.get_ip_from_container('yolo') + assert 'Could not determine ip address for container' in str(exc_info.value) diff --git a/tests/unit/hooks/data_source/test_utils.py b/tests/unit/hooks/data_source/test_utils.py deleted file mode 100644 index 0db4c7ec..00000000 --- a/tests/unit/hooks/data_source/test_utils.py +++ /dev/null @@ -1,93 +0,0 @@ -from subprocess import CalledProcessError - -import pytest -from flexmock import flexmock - -from borgmatic.hooks.data_source import utils - - -def test_get_database_option(): - data_source = {'option': 'original_value', 'restore_option': 'restore_value'} - connection_params = {'option': 'connection_value'} - - assert utils.resolve_database_option('option', data_source) == 'original_value' - assert utils.resolve_database_option('option', data_source, restore=True) == 'restore_value' - assert ( - utils.resolve_database_option('option', data_source, connection_params) - == 'connection_value' - ) - assert ( - utils.resolve_database_option('option', data_source, connection_params, restore=True) - == 'connection_value' - ) - - -def test_get_hostname_option_via_container(): - data_source = { - 'container': 'original_container', - 'hostname': 'original_hostname', - 'restore_container': 'restore_container', - 'restore_hostname': 'restore_hostname', - } - connection_params = {'container': 'connection_container', 'hostname': 'connection_hostname'} - - flexmock(utils).should_receive('get_ip_from_container').with_args( - 'original_container' - ).and_return('container_ip_1') - flexmock(utils).should_receive('get_ip_from_container').with_args( - 'connection_container' - ).and_return('container_ip_2') - flexmock(utils).should_receive('get_ip_from_container').with_args( - 'restore_container' - ).and_return('container_ip_3') - - assert utils.resolve_database_option('hostname', data_source) == 'container_ip_1' - assert ( - utils.resolve_database_option('hostname', data_source, connection_params) - == 'container_ip_2' - ) - assert utils.resolve_database_option('hostname', data_source, restore=True) == 'container_ip_3' - - -def test_get_container_ip_without_engines(): - flexmock(utils.shutil).should_receive('which').and_return(None).and_return(None) - - with pytest.raises(ValueError): - utils.get_ip_from_container('yolo') - - -def test_get_container_ip_success(): - flexmock(utils.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman') - - flexmock(utils).should_receive('execute_command_and_capture_output').and_return( - '{"IPAddress": "1.2.3.4"}' - ) - - addr = utils.get_ip_from_container('yolo') - assert addr == '1.2.3.4' - - flexmock(utils).should_receive('execute_command_and_capture_output').and_return( - '{"Networks": {"my_network": {"IPAddress": "5.6.7.8"}}}' - ) - - assert utils.get_ip_from_container('yolo') == '5.6.7.8' - - -def test_get_container_ip_container_not_found(): - flexmock(utils.shutil).should_receive('which').and_return('/usr/bin/podman') - flexmock(utils).should_receive('execute_command_and_capture_output').and_raise( - CalledProcessError, 1, ['/usr/bin/podman', 'inspect', 'yolo'], None, 'No such object' - ) - - with pytest.raises(CalledProcessError): - utils.get_ip_from_container('does not exist') - - -def test_get_container_ip_container_no_network(): - flexmock(utils.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman') - - flexmock(utils).should_receive('execute_command_and_capture_output').and_return('{}') - - with pytest.raises(ValueError) as exc_info: - utils.get_ip_from_container('yolo') - assert 'Could not determine ip address for container' in str(exc_info.value) From b306cac5c1c2cb11d28544c3705cd4aaba01b42b Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 31 Jul 2025 08:06:07 +0200 Subject: [PATCH 11/21] raise ValueError if json decode fails when getting the container ip --- borgmatic/hooks/data_source/config.py | 5 ++++- tests/unit/hooks/data_source/test_config.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/borgmatic/hooks/data_source/config.py b/borgmatic/hooks/data_source/config.py index 194ec1be..9a794853 100644 --- a/borgmatic/hooks/data_source/config.py +++ b/borgmatic/hooks/data_source/config.py @@ -64,7 +64,10 @@ def get_ip_from_container(container): logger.debug(f"Could not find container '{container}' with engine '{engine}'") continue # Container does not exist - network_data = json.loads(output.strip()) + try: + network_data = json.loads(output.strip()) + except json.JSONDecodeError as e: + raise ValueError(f'Could not decode JSON output from {engine}') from e main_ip = network_data.get('IPAddress') if main_ip: return main_ip diff --git a/tests/unit/hooks/data_source/test_config.py b/tests/unit/hooks/data_source/test_config.py index ae3198d0..5b583231 100644 --- a/tests/unit/hooks/data_source/test_config.py +++ b/tests/unit/hooks/data_source/test_config.py @@ -91,3 +91,13 @@ def test_get_container_ip_container_no_network(): with pytest.raises(ValueError) as exc_info: config.get_ip_from_container('yolo') assert 'Could not determine ip address for container' in str(exc_info.value) + + +def test_get_container_ip_broken_output(): + flexmock(config.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman') + + flexmock(config).should_receive('execute_command_and_capture_output').and_return('abc') + + with pytest.raises(ValueError) as exc_info: + config.get_ip_from_container('yolo') + assert 'Could not decode JSON output' in str(exc_info.value) From eef264dcc50bb3c43378509a481510a47b107c37 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 31 Jul 2025 08:29:29 +0200 Subject: [PATCH 12/21] test improvements --- .../hooks/data_source/test_database.py | 135 ++++++++++++------ 1 file changed, 94 insertions(+), 41 deletions(-) diff --git a/tests/end-to-end/hooks/data_source/test_database.py b/tests/end-to-end/hooks/data_source/test_database.py index 74030130..5e38fdde 100644 --- a/tests/end-to-end/hooks/data_source/test_database.py +++ b/tests/end-to-end/hooks/data_source/test_database.py @@ -21,7 +21,6 @@ def write_configuration( postgresql_all_dump_format=None, mariadb_mysql_all_dump_format=None, mongodb_dump_format='archive', - use_containers=False, ): ''' Write out borgmatic configuration into a file at the config path. Set the options so as to work @@ -35,7 +34,79 @@ def write_configuration( f'format: {mariadb_mysql_all_dump_format}' if mariadb_mysql_all_dump_format else '' ) - hostname_option = 'container' if use_containers else 'hostname' + config_yaml = f''' +source_directories: + - {source_directory} +repositories: + - path: {repository_path} +user_runtime_directory: {user_runtime_directory} + +encryption_passphrase: "test" + +postgresql_databases: + - name: test + hostname: postgresql + username: postgres + password: test + format: {postgresql_dump_format} + - name: all + {postgresql_all_format_option} + hostname: postgresql + username: postgres + password: test +mariadb_databases: + - name: test + hostname: mariadb + username: root + password: test + - name: all + {mariadb_mysql_dump_format_option} + hostname: mariadb + username: root + password: test +mysql_databases: + - name: test + hostname: not-actually-mysql + username: root + password: test + - name: all + {mariadb_mysql_dump_format_option} + hostname: not-actually-mysql + username: root + password: test +mongodb_databases: + - name: test + hostname: mongodb + username: root + password: test + authentication_database: admin + format: {mongodb_dump_format} + - name: all + hostname: mongodb + username: root + password: test +sqlite_databases: + - name: sqlite_test + path: /tmp/sqlite_test.db +''' + + with open(config_path, 'w') as config_file: + config_file.write(config_yaml) + + return ruamel.yaml.YAML(typ='safe').load(config_yaml) + + +def write_container_configuration( + source_directory, + config_path, + repository_path, + user_runtime_directory, +): + ''' + Write out borgmatic configuration into a file at the config path. Set the options so as to work + for testing. This includes injecting the given repository path, borgmatic source directory for + storing database dumps, and encryption passphrase. + ''' config_yaml = f''' source_directories: @@ -48,46 +119,32 @@ encryption_passphrase: "test" postgresql_databases: - name: test - {hostname_option}: postgresql - username: postgres - password: test - format: {postgresql_dump_format} - - name: all - {postgresql_all_format_option} - {hostname_option}: postgresql + hostname: postgresql username: postgres password: test + restore_hostname: postgresql2 + restore_port: 5433 + restore_password: test2 mariadb_databases: - name: test - {hostname_option}: mariadb - username: root - password: test - - name: all - {mariadb_mysql_dump_format_option} - {hostname_option}: mariadb + hostname: mariadb username: root password: test + restore_hostname: mariadb2 + restore_port: 3307 + restore_username: root + restore_password: test2 mysql_databases: - name: test - {hostname_option}: not-actually-mysql - username: root - password: test - - name: all - {mariadb_mysql_dump_format_option} - {hostname_option}: not-actually-mysql + hostname: not-actually-mysql username: root password: test mongodb_databases: - name: test - {hostname_option}: mongodb + hostname: mongodb username: root password: test authentication_database: admin - format: {mongodb_dump_format} - - name: all - {hostname_option}: mongodb - username: root - password: test sqlite_databases: - name: sqlite_test path: /tmp/sqlite_test.db @@ -115,7 +172,6 @@ def write_custom_restore_configuration( postgresql_all_dump_format=None, mariadb_mysql_all_dump_format=None, mongodb_dump_format='archive', - use_containers=False, ): ''' Write out borgmatic configuration into a file at the config path. Set the options so as to work @@ -123,8 +179,6 @@ def write_custom_restore_configuration( restore_username, restore_password and restore_path. ''' - hostname_option = 'container' if use_containers else 'hostname' - config_yaml = f''' source_directories: - {source_directory} @@ -136,39 +190,39 @@ encryption_passphrase: "test" postgresql_databases: - name: test - {hostname_option}: postgresql + hostname: postgresql username: postgres password: test format: {postgresql_dump_format} - restore_{hostname_option}: postgresql2 + restore_hostname: postgresql2 restore_port: 5433 restore_password: test2 mariadb_databases: - name: test - {hostname_option}: mariadb + hostname: mariadb username: root password: test - restore_{hostname_option}: mariadb2 + restore_hostname: mariadb2 restore_port: 3307 restore_username: root restore_password: test2 mysql_databases: - name: test - {hostname_option}: not-actually-mysql + hostname: not-actually-mysql username: root password: test - restore_{hostname_option}: not-actually-mysql2 + restore_hostname: not-actually-mysql2 restore_port: 3307 restore_username: root restore_password: test2 mongodb_databases: - name: test - {hostname_option}: mongodb + hostname: mongodb username: root password: test authentication_database: admin format: {mongodb_dump_format} - restore_{hostname_option}: mongodb2 + restore_hostname: mongodb2 restore_port: 27018 restore_username: root2 restore_password: test2 @@ -673,12 +727,11 @@ def test_database_dump_and_restore_containers(): try: config_path = os.path.join(temporary_directory, 'test.yaml') - config = write_configuration( + config = write_container_configuration( temporary_directory, config_path, repository_path, temporary_directory, - use_containers=True, ) create_test_tables(config) select_test_tables(config) @@ -716,7 +769,7 @@ def test_database_dump_and_restore_containers(): ) # Ensure the test tables have actually been restored. - select_test_tables(config) + select_test_tables(config, use_restore_options=True) finally: os.chdir(original_working_directory) shutil.rmtree(temporary_directory) From 8639b73f8070946c645f86d6e589d3ebedd19364 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 31 Jul 2025 09:23:54 +0200 Subject: [PATCH 13/21] fix label handling and improve unittests --- borgmatic/actions/restore.py | 8 ++-- tests/unit/actions/test_restore.py | 67 ++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index c325454b..21ce7e5d 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -35,9 +35,11 @@ def dumps_match(first, second, default_port=None): ''' # label kinda counts as id, if they match ignore hostname & port if first.label not in {None, UNSPECIFIED} and first.label == second.label: - return first[:2] == second[:2] + field_list = ('hook_name', 'data_source_name') + else: + field_list = Dump._fields - for field_name in first._fields: + for field_name in field_list: first_value = getattr(first, field_name) second_value = getattr(second, field_name) @@ -422,7 +424,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive): data_source_name=name, hostname=restore_arguments.original_hostname or UNSPECIFIED, port=restore_arguments.original_port, - label=restore_arguments.original_label, + label=restore_arguments.original_label or UNSPECIFIED, ) for name in restore_arguments.data_sources or (UNSPECIFIED,) } diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index ec8488b0..f1b5d891 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -132,6 +132,12 @@ import borgmatic.actions.restore as module 5432, True, ), + ( + module.Dump('postgresql_databases', 'foo', 'some_host1', 5433, 'unique'), + module.Dump(module.UNSPECIFIED, 'foo', 'some_host2', None, 'unique'), + 5432, + True, + ), ), ) def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values( @@ -730,11 +736,15 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive(): } flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED), + module.Dump( + module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'foo'), ).and_return(True) flexmock(module).should_receive('dumps_match').with_args( - module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED), + module.Dump( + module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'bar'), ).and_return(True) @@ -758,9 +768,15 @@ def test_get_dumps_to_restore_raises_for_requested_dumps_missing_from_archive(): module.Dump('postgresql_databases', 'foo'), } flexmock(module).should_receive('dumps_match').and_return(False) - flexmock(module).should_receive('render_dump_metadata').and_return('test') + flexmock(module).should_receive('dumps_match').with_args( + module.Dump( + module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), + module.Dump('postgresql_databases', 'foo'), + ).and_return(True) + flexmock(module).should_receive('render_dump_metadata').and_return('test').once() - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc_info: module.get_dumps_to_restore( restore_arguments=flexmock( hook=None, @@ -771,6 +787,7 @@ def test_get_dumps_to_restore_raises_for_requested_dumps_missing_from_archive(): ), dumps_from_archive=dumps_from_archive, ) + assert 'dump test missing from archive' in str(exc_info.value) def test_get_dumps_to_restore_without_requested_dumps_finds_all_archive_dumps(): @@ -832,11 +849,15 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques } flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED), + module.Dump( + module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'foo'), ).and_return(True) flexmock(module).should_receive('dumps_match').with_args( - module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED), + module.Dump( + module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'bar'), ).and_return(True) @@ -858,16 +879,20 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques def test_get_dumps_to_restore_raises_for_multiple_matching_dumps_in_archive(): flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED), + module.Dump( + module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'foo'), ).and_return(True) flexmock(module).should_receive('dumps_match').with_args( - module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED), + module.Dump( + module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('mariadb_databases', 'foo'), ).and_return(True) flexmock(module).should_receive('render_dump_metadata').and_return('test') - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc_info: module.get_dumps_to_restore( restore_arguments=flexmock( hook=None, @@ -881,17 +906,20 @@ def test_get_dumps_to_restore_raises_for_multiple_matching_dumps_in_archive(): module.Dump('mariadb_databases', 'foo'), }, ) + assert 'Try adding flags to disambiguate.' in str(exc_info.value) def test_get_dumps_to_restore_raises_for_all_in_requested_dumps_and_requested_dumps_missing_from_archive(): flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED), + module.Dump( + module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'foo'), ).and_return(True) - flexmock(module).should_receive('render_dump_metadata').and_return('test') + flexmock(module).should_receive('render_dump_metadata').and_return('test').once() - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc_info: module.get_dumps_to_restore( restore_arguments=flexmock( hook=None, @@ -900,8 +928,9 @@ def test_get_dumps_to_restore_raises_for_all_in_requested_dumps_and_requested_du original_port=None, original_label=None, ), - dumps_from_archive={module.Dump('postresql_databases', 'foo')}, + dumps_from_archive={module.Dump('postgresql_databases', 'foo')}, ) + assert 'dump test missing from archive' in str(exc_info.value) def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_archive(): @@ -912,7 +941,9 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar } flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump('postgresql_databases', 'foo', hostname=module.UNSPECIFIED), + module.Dump( + 'postgresql_databases', 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'foo'), ).and_return(True) @@ -938,7 +969,9 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f } flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump('postgresql_databases', 'foo', hostname=module.UNSPECIFIED), + module.Dump( + 'postgresql_databases', 'foo', hostname=module.UNSPECIFIED, label=module.UNSPECIFIED + ), module.Dump('postgresql_databases', 'foo'), ).and_return(True) @@ -964,7 +997,7 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc } flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump('postgresql_databases', 'foo', 'host'), + module.Dump('postgresql_databases', 'foo', 'host', label=module.UNSPECIFIED), module.Dump('postgresql_databases', 'foo', 'host'), ).and_return(True) @@ -990,7 +1023,7 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive } flexmock(module).should_receive('dumps_match').and_return(False) flexmock(module).should_receive('dumps_match').with_args( - module.Dump('postgresql_databases', 'foo', 'host', 1234), + module.Dump('postgresql_databases', 'foo', 'host', 1234, label=module.UNSPECIFIED), module.Dump('postgresql_databases', 'foo', 'host', 1234), ).and_return(True) From d44fb246702d6526392015187ae93efe90125dff Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Tue, 19 Aug 2025 21:10:26 +0200 Subject: [PATCH 14/21] get `--container` arg working --- borgmatic/config/arguments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index e50314ae..295f48c5 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -146,9 +146,9 @@ def prepare_arguments_for_config(global_arguments, schema): keys = tuple(argument_name.split('.')) option_type = type_for_option(schema, keys) - # The argument doesn't correspond to any option in the schema, so ignore it. It's - # probably a flag that borgmatic has on the command-line but not in configuration. - if option_type is None: + # The argument doesn't correspond to any option in the schema, or it is a complex argument, so ignore it. + # It's probably a flag that borgmatic has on the command-line but not in configuration. + if option_type in {'object', None}: continue prepared_values.append( From 16c8098b0623ee3e333a30d839a9918b78353011 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Fri, 22 Aug 2025 14:31:39 +0200 Subject: [PATCH 15/21] address some review comments --- borgmatic/commands/arguments.py | 2 +- borgmatic/hooks/data_source/config.py | 12 +++++------- borgmatic/hooks/data_source/mariadb.py | 22 ++++++++++++++-------- borgmatic/hooks/data_source/postgresql.py | 4 ++-- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 9292300f..aac92c8b 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1493,7 +1493,7 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 ) restore_group.add_argument( '--original-label', - help='The label where to dump to restore came from, only necessary if you need to disambiguate dumps', + help='The label where the dump to restore came from, only necessary if you need to disambiguate dumps', ) restore_group.add_argument( '--original-hostname', diff --git a/borgmatic/hooks/data_source/config.py b/borgmatic/hooks/data_source/config.py index 9a794853..7deaa201 100644 --- a/borgmatic/hooks/data_source/config.py +++ b/borgmatic/hooks/data_source/config.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) def resolve_database_option(option, data_source, connection_params=None, restore=False): # Special case `hostname` since it overlaps with `container` if option == 'hostname': - return _get_hostname_from_config(data_source, connection_params, restore) + return get_hostname_from_config(data_source, connection_params, restore) if connection_params and (value := connection_params.get(option)): return value if restore and f'restore_{option}' in data_source: @@ -21,7 +21,7 @@ def resolve_database_option(option, data_source, connection_params=None, restore return data_source.get(option) -def _get_hostname_from_config(data_source, connection_params=None, restore=False): +def get_hostname_from_config(data_source, connection_params=None, restore=False): # connection params win, full stop if connection_params: if container := connection_params.get('container'): @@ -37,7 +37,7 @@ def _get_hostname_from_config(data_source, connection_params=None, restore=False # ... and finally fall back to the normal options if 'container' in data_source: return get_ip_from_container(data_source['container']) - return data_source.get('hostname', '') + return data_source.get('hostname') def get_ip_from_container(container): @@ -68,13 +68,11 @@ def get_ip_from_container(container): network_data = json.loads(output.strip()) except json.JSONDecodeError as e: raise ValueError(f'Could not decode JSON output from {engine}') from e - main_ip = network_data.get('IPAddress') - if main_ip: + if main_ip := network_data.get('IPAddress'): return main_ip # No main IP found, try the networks for network in network_data.get('Networks', {}).values(): - ip = network.get('IPAddress') - if ip: + if ip := network.get('IPAddress'): return ip if last_error: diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 774a0d4a..5a229711 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -12,7 +12,7 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import config as ds_config +from borgmatic.hooks.data_source import config as database_config from borgmatic.hooks.data_source import dump logger = logging.getLogger(__name__) @@ -123,7 +123,7 @@ def database_names_to_dump(database, config, username, password, environment, dr ) extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) password_transport = database.get('password_transport', 'pipe') - hostname = ds_config.resolve_database_option('hostname', database) + hostname = database_config.resolve_database_option('hostname', database) show_command = ( mariadb_show_command + ( @@ -195,7 +195,7 @@ def execute_dump_command( ) extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) password_transport = database.get('password_transport', 'pipe') - hostname = ds_config.resolve_database_option('hostname', database) + hostname = database_config.resolve_database_option('hostname', database) dump_command = ( mariadb_dump_command + ( @@ -418,17 +418,23 @@ def restore_data_source_dump( subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = ds_config.resolve_database_option( + hostname = database_config.resolve_database_option( 'hostname', data_source, connection_params, restore=True ) - port = ds_config.resolve_database_option('port', data_source, connection_params, restore=True) - tls = ds_config.resolve_database_option('tls', data_source, restore=True) + port = database_config.resolve_database_option( + 'port', data_source, connection_params, restore=True + ) + tls = database_config.resolve_database_option('tls', data_source, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ds_config.resolve_database_option('username', data_source, connection_params, restore=True), + database_config.resolve_database_option( + 'username', data_source, connection_params, restore=True + ), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - ds_config.resolve_database_option('password', data_source, connection_params, restore=True), + database_config.resolve_database_option( + 'password', data_source, connection_params, restore=True + ), config, ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 3da88345..b6105bb8 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -349,8 +349,8 @@ def restore_data_source_dump( dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), data_source['name'], - hostname=data_source.get('hostname'), - port=data_source.get('port'), + hostname=hostname, + port=port, label=data_source.get('label', data_source.get('container')), ) psql_command = tuple( From 56a3f6d854642da9b761e7d226378e57b19841dc Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Fri, 22 Aug 2025 14:48:32 +0200 Subject: [PATCH 16/21] address more review comments --- .../hooks/data_source/test_database.py | 17 +++++++++++------ tests/unit/config/test_arguments.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/end-to-end/hooks/data_source/test_database.py b/tests/end-to-end/hooks/data_source/test_database.py index 5e38fdde..4eb8cbd1 100644 --- a/tests/end-to-end/hooks/data_source/test_database.py +++ b/tests/end-to-end/hooks/data_source/test_database.py @@ -9,8 +9,6 @@ import pymongo import pytest import ruamel.yaml -from borgmatic.hooks.data_source import config - def write_configuration( source_directory, @@ -275,10 +273,17 @@ postgresql_databases: def get_connection_params(database, use_restore_options=False): - hostname = config.resolve_database_option('hostname', database, restore=use_restore_options) - port = config.resolve_database_option('port', database, restore=use_restore_options) - username = config.resolve_database_option('username', database, restore=use_restore_options) - password = config.resolve_database_option('password', database, restore=use_restore_options) + hostname = (database.get('restore_hostname') if use_restore_options else None) or database.get( + 'container', + database.get('hostname'), + ) + port = (database.get('restore_port') if use_restore_options else None) or database.get('port') + username = (database.get('restore_username') if use_restore_options else None) or database.get( + 'username', + ) + password = (database.get('restore_password') if use_restore_options else None) or database.get( + 'password', + ) return (hostname, port, username, password) diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 92907206..d73c08e1 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -209,6 +209,19 @@ def test_prepare_arguments_for_config_skips_option_with_none_value(): ) == ((('other_option',), 'value2'),) +def test_prepare_arguments_for_config_skips_option_with_complex_schema(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(my_option='value1', other_option='value2'), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ((('other_option',), 'value2'),) + + def test_prepare_arguments_for_config_skips_option_missing_from_schema(): assert module.prepare_arguments_for_config( global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), From 0e90087dc4e558def848880f65705820de38394c Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Mon, 15 Sep 2025 20:24:44 +0200 Subject: [PATCH 17/21] Finalize container support --- borgmatic/actions/restore.py | 19 ++- borgmatic/commands/arguments.py | 4 + borgmatic/hooks/data_source/config.py | 19 +++ borgmatic/hooks/data_source/dump.py | 13 +- borgmatic/hooks/data_source/mariadb.py | 7 +- borgmatic/hooks/data_source/mongodb.py | 8 +- borgmatic/hooks/data_source/mysql.py | 7 +- borgmatic/hooks/data_source/postgresql.py | 8 +- borgmatic/hooks/data_source/sqlite.py | 3 +- .../hooks/data_source/test_database.py | 18 ++- tests/unit/actions/test_restore.py | 120 +++++++++++++++--- tests/unit/hooks/data_source/test_dump.py | 11 +- 12 files changed, 193 insertions(+), 44 deletions(-) diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 21ce7e5d..4f8b271a 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -22,8 +22,8 @@ UNSPECIFIED = object() Dump = collections.namedtuple( 'Dump', - ('hook_name', 'data_source_name', 'hostname', 'port', 'label'), - defaults=('localhost', None, None), + ('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'), + defaults=('localhost', None, None, None), ) @@ -33,7 +33,7 @@ def dumps_match(first, second, default_port=None): indicates that the field should match any value. If a default port is given, then consider any dump having that port to match with a dump having a None port. ''' - # label kinda counts as id, if they match ignore hostname & port + # label kinda counts as an unique id, if they match ignore host/container/port if first.label not in {None, UNSPECIFIED} and first.label == second.label: field_list = ('hook_name', 'data_source_name') else: @@ -65,15 +65,15 @@ def render_dump_metadata(dump): ''' label = dump.label or UNSPECIFIED name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name - hostname = dump.hostname or UNSPECIFIED + host = dump.container or dump.hostname or UNSPECIFIED port = None if dump.port is UNSPECIFIED else dump.port if label is not UNSPECIFIED: metadata = f'{name}@{label}' elif port: - metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}' + metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}' else: - metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}' + metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}' if dump.hook_name not in {None, UNSPECIFIED}: return f'{metadata} ({dump.hook_name})' @@ -110,7 +110,8 @@ def get_configured_data_source(config, restore_dump): hook_data_source.get('name'), hook_data_source.get('hostname', 'localhost'), hook_data_source.get('port'), - hook_data_source.get('label') or hook_data_source.get('container') or UNSPECIFIED, + hook_data_source.get('label') or UNSPECIFIED, + hook_data_source.get('container'), ), restore_dump, default_port, @@ -425,6 +426,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive): hostname=restore_arguments.original_hostname or UNSPECIFIED, port=restore_arguments.original_port, label=restore_arguments.original_label or UNSPECIFIED, + container=restore_arguments.original_container or UNSPECIFIED, ) for name in restore_arguments.data_sources or (UNSPECIFIED,) } @@ -433,6 +435,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive): or restore_arguments.original_hostname or restore_arguments.original_port or restore_arguments.original_label + or restore_arguments.original_container else { Dump( hook_name=UNSPECIFIED, @@ -440,6 +443,7 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive): hostname=UNSPECIFIED, port=UNSPECIFIED, label=UNSPECIFIED, + container=UNSPECIFIED, ), } ) @@ -585,6 +589,7 @@ def run_restore( restore_dump.hostname, restore_dump.port, restore_dump.label, + restore_dump.container, ), ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index aac92c8b..9f77a7a8 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1499,6 +1499,10 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 '--original-hostname', help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps', ) + restore_group.add_argument( + '--original-container', + help='The container where the dump to restore came from, only necessary if you need to disambiguate dumps', + ) restore_group.add_argument( '--original-port', type=int, diff --git a/borgmatic/hooks/data_source/config.py b/borgmatic/hooks/data_source/config.py index 7deaa201..17cb313a 100644 --- a/borgmatic/hooks/data_source/config.py +++ b/borgmatic/hooks/data_source/config.py @@ -11,6 +11,14 @@ logger = logging.getLogger(__name__) def resolve_database_option(option, data_source, connection_params=None, restore=False): + ''' + Resolves a database option from the given data source configuration dict and + connection parameters dict. If restore is set to True it will consider the + `restore_