add support for container names/id when dumping databases.

NEWS

@@ -1,4 +1,5 @@
 2.0.8.dev0
+ * #1116: Add support for dumping database containers via their container name.
  * #1116: Add support for database backup labels.
  * #1114: Document systemd configuration changes for the ZFS filesystem hook.
  * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in

borgmatic/config/schema.yaml

@@ -1328,6 +1328,12 @@ properties:
                     description: |
                         Label to identify the database dump in the backup.
                     example: my_backup_label
+                container:
+                    type: string
+                    description:
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
                 hostname:
                     type: string
                     description: |
@@ -1534,6 +1540,12 @@ properties:
                     description: |
                         Label to identify the database dump in the backup.
                     example: my_backup_label
+                container:
+                    type: string
+                    description:
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
                 hostname:
                     type: string
                     description: |
@@ -1704,6 +1716,12 @@ properties:
                     description: |
                         Label to identify the database dump in the backup.
                     example: my_backup_label
+                container:
+                    type: string
+                    description:
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
                 hostname:
                     type: string
                     description: |
@@ -1935,6 +1953,12 @@ properties:
                     description: |
                         Label to identify the database dump in the backup.
                     example: my_backup_label
+                container:
+                    type: string
+                    description:
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
                 hostname:
                     type: string
                     description: |

borgmatic/hooks/data_source/mariadb.py

@@ -12,7 +12,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
-from borgmatic.hooks.data_source import dump
+from borgmatic.hooks.data_source import dump, utils
 
 logger = logging.getLogger(__name__)
 
@@ -122,6 +122,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options'))
     password_transport = database.get('password_transport', 'pipe')
+    hostname = utils.get_hostname_from_config(database)
     show_command = (
         mariadb_show_command
         + (
@@ -130,9 +131,9 @@ def database_names_to_dump(database, config, username, password, environment, dr
             else ()
         )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -176,9 +177,9 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
-        database.get('port'),
-        database.get('label'),
+        hostname=database.get('hostname'),
+        port=database.get('port'),
+        label=database.get('label', database.get('container')),
     )
 
     if os.path.exists(dump_filename):
@@ -193,6 +194,7 @@ def execute_dump_command(
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('options'))
     password_transport = database.get('password_transport', 'pipe')
+    hostname = utils.get_hostname_from_config(database)
     dump_command = (
         mariadb_dump_command
         + (
@@ -202,9 +204,9 @@ def execute_dump_command(
         )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -417,7 +419,7 @@ def restore_data_source_dump(
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
     hostname = connection_params['hostname'] or data_source.get(
         'restore_hostname',
-        data_source.get('hostname'),
+        utils.get_hostname_from_config(data_source),
     )
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),

borgmatic/hooks/data_source/mongodb.py

@@ -6,7 +6,7 @@ import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
 from borgmatic.execute import execute_command, execute_command_with_processes
-from borgmatic.hooks.data_source import dump
+from borgmatic.hooks.data_source import dump, utils
 
 logger = logging.getLogger(__name__)
 
@@ -69,9 +69,9 @@ def dump_data_sources(
         dump_filename = dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            database.get('hostname'),
-            database.get('port'),
-            database.get('label'),
+            hostname=database.get('hostname'),
+            port=database.get('port'),
+            label=database.get('label', database.get('container')),
         )
         dump_format = database.get('format', 'archive')
 
@@ -139,10 +139,11 @@ def build_dump_command(database, config, dump_filename, dump_format):
     dump_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump')
     )
+    hostname = utils.get_hostname_from_config(database)
     return (
         dump_command
         + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
-        + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+        + (('--host', shlex.quote(hostname)) if hostname else ())
         + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
         + (
             (
@@ -238,8 +239,9 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
-        data_source.get('label'),
+        hostname=data_source.get('hostname'),
+        port=data_source.get('port'),
+        label=data_source.get('label', data_source.get('container')),
     )
     restore_command = build_restore_command(
         extract_process,
@@ -269,7 +271,7 @@ def build_restore_command(extract_process, database, config, dump_filename, conn
     '''
     hostname = connection_params['hostname'] or database.get(
         'restore_hostname',
-        database.get('hostname'),
+        utils.get_hostname_from_config(database),
     )
     port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
     username = borgmatic.hooks.credential.parse.resolve_credential(

borgmatic/hooks/data_source/mysql.py

@@ -12,7 +12,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
-from borgmatic.hooks.data_source import dump
+from borgmatic.hooks.data_source import dump, utils
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +47,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options'))
     )
     password_transport = database.get('password_transport', 'pipe')
+    hostname = utils.get_hostname_from_config(database)
     show_command = (
         mysql_show_command
         + (
@@ -59,9 +60,9 @@ def database_names_to_dump(database, config, username, password, environment, dr
             else ()
         )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -102,9 +103,9 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
-        database.get('port'),
-        database.get('label'),
+        hostname=database.get('hostname'),
+        port=database.get('port'),
+        label=database.get('label', database.get('container')),
     )
 
     if os.path.exists(dump_filename):
@@ -120,6 +121,7 @@ def execute_dump_command(
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options'))
     )
     password_transport = database.get('password_transport', 'pipe')
+    hostname = utils.get_hostname_from_config(database)
     dump_command = (
         mysql_dump_command
         + (
@@ -133,9 +135,9 @@ def execute_dump_command(
         )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -348,7 +350,7 @@ def restore_data_source_dump(
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
     hostname = connection_params['hostname'] or data_source.get(
         'restore_hostname',
-        data_source.get('hostname'),
+        utils.get_hostname_from_config(data_source),
     )
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),

borgmatic/hooks/data_source/postgresql.py

@@ -13,7 +13,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
-from borgmatic.hooks.data_source import dump
+from borgmatic.hooks.data_source import dump, utils
 
 logger = logging.getLogger(__name__)
 
@@ -91,10 +91,11 @@ def database_names_to_dump(database, config, environment, dry_run):
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql')
     )
+    hostname = utils.get_hostname_from_config(database)
     list_command = (
         psql_command
         + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (
             (
@@ -184,9 +185,9 @@ def dump_data_sources(
             dump_filename = dump.make_data_source_dump_filename(
                 dump_path,
                 database_name,
-                database.get('hostname'),
-                database.get('port'),
-                database.get('label'),
+                hostname=database.get('hostname'),
+                port=database.get('port'),
+                label=database.get('label', database.get('container')),
             )
 
             if os.path.exists(dump_filename):
@@ -195,6 +196,7 @@ def dump_data_sources(
                 )
                 continue
 
+            hostname = utils.get_hostname_from_config(database)
             command = (
                 dump_command
                 + (
@@ -202,7 +204,7 @@ def dump_data_sources(
                     '--clean',
                     '--if-exists',
                 )
-                + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+                + (('--host', shlex.quote(hostname)) if hostname else ())
                 + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
                 + (
                     (
@@ -342,7 +344,7 @@ def restore_data_source_dump(
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
     hostname = connection_params['hostname'] or data_source.get(
         'restore_hostname',
-        data_source.get('hostname'),
+        utils.get_hostname_from_config(data_source),
     )
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),
@@ -359,8 +361,9 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
-        data_source.get('label'),
+        hostname=data_source.get('hostname'),
+        port=data_source.get('port'),
+        label=data_source.get('label', data_source.get('container')),
     )
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql')

borgmatic/hooks/data_source/utils.py

@@ -0,0 +1,47 @@
+import json
+import shutil
+import subprocess
+
+from borgmatic.execute import execute_command_and_capture_output
+
+IS_A_HOOK = False
+
+
+def get_hostname_from_config(database):
+    if 'container' in database:
+        return get_ip_from_container(database['container'])
+    return database.get('hostname', '')
+
+
+def get_ip_from_container(container):
+    engines = (shutil.which(engine) for engine in ('docker', 'podman'))
+    engines = [engine for engine in engines if engine]
+
+    if not engines:
+        raise xxx  # TODO: What to raise here, tell the user to install docker/podman
+
+    for engine in engines:
+        try:
+            output = execute_command_and_capture_output(
+                (
+                    engine,
+                    'container',
+                    'inspect',
+                    '--format={{json .NetworkSettings}}',
+                    container,
+                )
+            )
+        except subprocess.CalledProcessError:
+            continue  # Container does not exist
+
+        network_data = json.loads(output.strip())
+        main_ip = network_data.get('IPAddress')
+        if main_ip:
+            return main_ip
+        # No main IP found, try the networks
+        for network in network_data.get('Networks', {}).values():
+            ip = network.get('IPAddress')
+            if ip:
+                return ip
+
+    raise xxx  # No container ip found, what to raise here

docs/how-to/backup-your-databases.md

@@ -214,35 +214,20 @@ these options in the `hooks:` section of your configuration.
 
 #### Database client on the host
 
-But what if borgmatic is running on the host? You can still connect to a
-database server container if its ports are properly exposed to the host. For
-instance, when running the database container, you can specify `--publish
-127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433
-on the host (only reachable on localhost, in this case). Or the same thing with
-Docker Compose:
-
-```yaml
-services:
-   your-database-server-container-name:
-       image: postgres
-       ports:
-           - 127.0.0.1:5433:5432
-```
-
-And then you can configure borgmatic running on the host to connect to the
-database:
+But what if borgmatic is running on the host? You can connect to the database
+container by specifying its container name or id:
 
 ```yaml
 postgresql_databases:
     - name: users
-      hostname: 127.0.0.1
+      container: your-database-server-container-name
       port: 5433
       username: postgres
       password: trustsome1
 ```
 
-Alter the ports in these examples to suit your particular database system.
-
+Now borgmatic will use the `docker`/`podman` CLI to figure out the container IP.
+Alternatively you can publish your container ports to the host.
 
 #### Database client in a running container
 

tests/unit/hooks/data_source/test_dump.py

@@ -25,9 +25,11 @@ def test_make_data_source_dump_filename_uses_name_and_hostname_and_port():
     )
 
 
-def test_make_data_source_dump_filename_users_label():
+def test_make_data_source_dump_filename_uses_label():
     assert (
-        module.make_data_source_dump_filename('databases', 'test', 'hostname', 1234, 'custom_label')
+        module.make_data_source_dump_filename(
+            'databases', 'test', 'hostname', 1234, label='custom_label'
+        )
         == 'databases/custom_label/test'
     )