From 133a11c647c5ad482d7b29c81ce18d5ec0fb56bb Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 24 Jul 2025 08:51:39 +0200 Subject: [PATCH] add support for container names/id when dumping databases. --- NEWS | 1 + borgmatic/config/schema.yaml | 24 ++++++++++++ borgmatic/hooks/data_source/mariadb.py | 20 +++++----- borgmatic/hooks/data_source/mongodb.py | 18 +++++---- borgmatic/hooks/data_source/mysql.py | 20 +++++----- borgmatic/hooks/data_source/postgresql.py | 21 +++++----- borgmatic/hooks/data_source/utils.py | 47 +++++++++++++++++++++++ docs/how-to/backup-your-databases.md | 25 +++--------- tests/unit/hooks/data_source/test_dump.py | 6 ++- 9 files changed, 125 insertions(+), 57 deletions(-) create mode 100644 borgmatic/hooks/data_source/utils.py diff --git a/NEWS b/NEWS index 639b9186..762cebd3 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 2.0.8.dev0 + * #1116: Add support for dumping database containers via their container name. * #1116: Add support for database backup labels. * #1114: Document systemd configuration changes for the ZFS filesystem hook. * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 5562c428..f53482c6 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1328,6 +1328,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | @@ -1534,6 +1540,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | @@ -1704,6 +1716,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | @@ -1935,6 +1953,12 @@ properties: description: | Label to identify the database dump in the backup. example: my_backup_label + container: + type: string + description: + Container name/id to connect to. When specified the + hostname is ignored. Requires docker/podman CLI. + example: debian_stable hostname: type: string description: | diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 4bd10bab..8d26d23e 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -12,7 +12,7 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -122,6 +122,7 @@ def database_names_to_dump(database, config, username, password, environment, dr ) extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) show_command = ( mariadb_show_command + ( @@ -130,9 +131,9 @@ def database_names_to_dump(database, config, username, password, environment, dr else () ) + extra_options - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -176,9 +177,9 @@ def execute_dump_command( dump_filename = dump.make_data_source_dump_filename( dump_path, database['name'], - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) if os.path.exists(dump_filename): @@ -193,6 +194,7 @@ def execute_dump_command( ) extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) dump_command = ( mariadb_dump_command + ( @@ -202,9 +204,9 @@ def execute_dump_command( ) + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -417,7 +419,7 @@ def restore_data_source_dump( dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( 'restore_hostname', - data_source.get('hostname'), + utils.get_hostname_from_config(data_source), ) port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 528ad607..c940ca10 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -6,7 +6,7 @@ import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.hooks.credential.parse from borgmatic.execute import execute_command, execute_command_with_processes -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -69,9 +69,9 @@ def dump_data_sources( dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), name, - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) dump_format = database.get('format', 'archive') @@ -139,10 +139,11 @@ def build_dump_command(database, config, dump_filename, dump_format): dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump') ) + hostname = utils.get_hostname_from_config(database) return ( dump_command + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) - + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + + (('--host', shlex.quote(hostname)) if hostname else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + ( ( @@ -238,8 +239,9 @@ def restore_data_source_dump( dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), data_source['name'], - data_source.get('hostname'), - data_source.get('label'), + hostname=data_source.get('hostname'), + port=data_source.get('port'), + label=data_source.get('label', data_source.get('container')), ) restore_command = build_restore_command( extract_process, @@ -269,7 +271,7 @@ def build_restore_command(extract_process, database, config, dump_filename, conn ''' hostname = connection_params['hostname'] or database.get( 'restore_hostname', - database.get('hostname'), + utils.get_hostname_from_config(database), ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) username = borgmatic.hooks.credential.parse.resolve_credential( diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 48991902..34e18ea2 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -12,7 +12,7 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -47,6 +47,7 @@ def database_names_to_dump(database, config, username, password, environment, dr borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) ) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) show_command = ( mysql_show_command + ( @@ -59,9 +60,9 @@ def database_names_to_dump(database, config, username, password, environment, dr else () ) + extra_options - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -102,9 +103,9 @@ def execute_dump_command( dump_filename = dump.make_data_source_dump_filename( dump_path, database['name'], - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) if os.path.exists(dump_filename): @@ -120,6 +121,7 @@ def execute_dump_command( borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) ) password_transport = database.get('password_transport', 'pipe') + hostname = utils.get_hostname_from_config(database) dump_command = ( mysql_dump_command + ( @@ -133,9 +135,9 @@ def execute_dump_command( ) + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--protocol', 'tcp') if hostname or 'port' in database else ()) + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) @@ -348,7 +350,7 @@ def restore_data_source_dump( dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( 'restore_hostname', - data_source.get('hostname'), + utils.get_hostname_from_config(data_source), ) port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 695d79d9..f3e41dc4 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -13,7 +13,7 @@ from borgmatic.execute import ( execute_command_and_capture_output, execute_command_with_processes, ) -from borgmatic.hooks.data_source import dump +from borgmatic.hooks.data_source import dump, utils logger = logging.getLogger(__name__) @@ -91,10 +91,11 @@ def database_names_to_dump(database, config, environment, dry_run): psql_command = tuple( shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql') ) + hostname = utils.get_hostname_from_config(database) list_command = ( psql_command + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') - + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--host', hostname) if hostname else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + ( ( @@ -184,9 +185,9 @@ def dump_data_sources( dump_filename = dump.make_data_source_dump_filename( dump_path, database_name, - database.get('hostname'), - database.get('port'), - database.get('label'), + hostname=database.get('hostname'), + port=database.get('port'), + label=database.get('label', database.get('container')), ) if os.path.exists(dump_filename): @@ -195,6 +196,7 @@ def dump_data_sources( ) continue + hostname = utils.get_hostname_from_config(database) command = ( dump_command + ( @@ -202,7 +204,7 @@ def dump_data_sources( '--clean', '--if-exists', ) - + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + + (('--host', shlex.quote(hostname)) if hostname else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + ( ( @@ -342,7 +344,7 @@ def restore_data_source_dump( dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( 'restore_hostname', - data_source.get('hostname'), + utils.get_hostname_from_config(data_source), ) port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), @@ -359,8 +361,9 @@ def restore_data_source_dump( dump_filename = dump.make_data_source_dump_filename( make_dump_path(borgmatic_runtime_directory), data_source['name'], - data_source.get('hostname'), - data_source.get('label'), + hostname=data_source.get('hostname'), + port=data_source.get('port'), + label=data_source.get('label', data_source.get('container')), ) psql_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql') diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/utils.py new file mode 100644 index 00000000..5709c228 --- /dev/null +++ b/borgmatic/hooks/data_source/utils.py @@ -0,0 +1,47 @@ +import json +import shutil +import subprocess + +from borgmatic.execute import execute_command_and_capture_output + +IS_A_HOOK = False + + +def get_hostname_from_config(database): + if 'container' in database: + return get_ip_from_container(database['container']) + return database.get('hostname', '') + + +def get_ip_from_container(container): + engines = (shutil.which(engine) for engine in ('docker', 'podman')) + engines = [engine for engine in engines if engine] + + if not engines: + raise xxx # TODO: What to raise here, tell the user to install docker/podman + + for engine in engines: + try: + output = execute_command_and_capture_output( + ( + engine, + 'container', + 'inspect', + '--format={{json .NetworkSettings}}', + container, + ) + ) + except subprocess.CalledProcessError: + continue # Container does not exist + + network_data = json.loads(output.strip()) + main_ip = network_data.get('IPAddress') + if main_ip: + return main_ip + # No main IP found, try the networks + for network in network_data.get('Networks', {}).values(): + ip = network.get('IPAddress') + if ip: + return ip + + raise xxx # No container ip found, what to raise here diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 4b3dc742..dc67691e 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -214,35 +214,20 @@ these options in the `hooks:` section of your configuration. #### Database client on the host -But what if borgmatic is running on the host? You can still connect to a -database server container if its ports are properly exposed to the host. For -instance, when running the database container, you can specify `--publish -127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433 -on the host (only reachable on localhost, in this case). Or the same thing with -Docker Compose: - -```yaml -services: - your-database-server-container-name: - image: postgres - ports: - - 127.0.0.1:5433:5432 -``` - -And then you can configure borgmatic running on the host to connect to the -database: +But what if borgmatic is running on the host? You can connect to the database +container by specifying its container name or id: ```yaml postgresql_databases: - name: users - hostname: 127.0.0.1 + container: your-database-server-container-name port: 5433 username: postgres password: trustsome1 ``` -Alter the ports in these examples to suit your particular database system. - +Now borgmatic will use the `docker`/`podman` CLI to figure out the container IP. +Alternatively you can publish your container ports to the host. #### Database client in a running container diff --git a/tests/unit/hooks/data_source/test_dump.py b/tests/unit/hooks/data_source/test_dump.py index 0acc3f5d..a18fae3c 100644 --- a/tests/unit/hooks/data_source/test_dump.py +++ b/tests/unit/hooks/data_source/test_dump.py @@ -25,9 +25,11 @@ def test_make_data_source_dump_filename_uses_name_and_hostname_and_port(): ) -def test_make_data_source_dump_filename_users_label(): +def test_make_data_source_dump_filename_uses_label(): assert ( - module.make_data_source_dump_filename('databases', 'test', 'hostname', 1234, 'custom_label') + module.make_data_source_dump_filename( + 'databases', 'test', 'hostname', 1234, label='custom_label' + ) == 'databases/custom_label/test' )