From e058df6b7ed08306af96f2b21cf0b0d787bf8cee Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 26 Jul 2025 21:22:32 +0200 Subject: [PATCH] cleanup database option resolving. --- borgmatic/commands/arguments.py | 4 ++ borgmatic/hooks/data_source/mariadb.py | 25 ++++-------- borgmatic/hooks/data_source/mongodb.py | 26 +++++-------- borgmatic/hooks/data_source/mysql.py | 25 ++++-------- borgmatic/hooks/data_source/postgresql.py | 47 ++++++++--------------- borgmatic/hooks/data_source/utils.py | 32 +++++++++++++-- 6 files changed, 75 insertions(+), 84 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e35cf5a5..cfa5da2c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1475,6 +1475,10 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 '--port', help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration', ) + restore_group.add_argument( + '--container', + help='Container to restore to. Defaults to the "restore_container" option in borgmatic\'s configuration', + ) restore_group.add_argument( '--username', help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration', diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 8d26d23e..95949c1d 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -122,7 +122,7 @@ def database_names_to_dump(database, config, username, password, environment, dr ) extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) show_command = ( mariadb_show_command + ( @@ -194,7 +194,7 @@ def execute_dump_command( ) extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) dump_command = ( mariadb_dump_command + ( @@ -417,26 +417,17 @@ def restore_data_source_dump( subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = connection_params['hostname'] or data_source.get( - 'restore_hostname', - utils.get_hostname_from_config(data_source), + hostname = utils.resolve_database_option( + 'hostname', data_source, connection_params, restore=True ) - port = str( - connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), - ) - tls = data_source.get('restore_tls', data_source.get('tls')) + port = utils.resolve_database_option('port', data_source, connection_params, restore=True) + tls = utils.resolve_database_option('tls', data_source, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) - ), + utils.resolve_database_option('username', data_source, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['password'] - or data_source.get('restore_password', data_source.get('password')) - ), + utils.resolve_database_option('password', data_source, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index c940ca10..a2d7d90f 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -139,7 +139,7 @@ def build_dump_command(database, config, dump_filename, dump_format): dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump') ) - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) return ( dump_command + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) @@ -160,7 +160,10 @@ def build_dump_command(database, config, dump_filename, dump_format): ) + (('--config', make_password_config_file(password)) if password else ()) + ( - ('--authenticationDatabase', shlex.quote(database['authentication_database'])) + ( + '--authenticationDatabase', + shlex.quote(database['authentication_database']), + ) if 'authentication_database' in database else () ) @@ -251,7 +254,7 @@ def restore_data_source_dump( connection_params, ) - logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}") + logger.debug(f'Restoring MongoDB database {data_source["name"]}{dry_run_label}') if dry_run: return @@ -269,23 +272,14 @@ def build_restore_command(extract_process, database, config, dump_filename, conn ''' Return the custom mongorestore_command from a single database configuration. ''' - hostname = connection_params['hostname'] or database.get( - 'restore_hostname', - utils.get_hostname_from_config(database), - ) - port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) + hostname = utils.resolve_database_option('hostname', database, connection_params, restore=True) + port = utils.resolve_database_option('port', database, connection_params, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or database.get('restore_username', database.get('username')) - ), + utils.resolve_database_option('username', database, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['password'] - or database.get('restore_password', database.get('password')) - ), + utils.resolve_database_option('password', database, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 34e18ea2..9553b0ed 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -47,7 +47,7 @@ def database_names_to_dump(database, config, username, password, environment, dr borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) ) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) show_command = ( mysql_show_command + ( @@ -121,7 +121,7 @@ def execute_dump_command( borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) ) password_transport = database.get('password_transport', 'pipe') - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) dump_command = ( mysql_dump_command + ( @@ -348,26 +348,17 @@ def restore_data_source_dump( subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = connection_params['hostname'] or data_source.get( - 'restore_hostname', - utils.get_hostname_from_config(data_source), + hostname = utils.resolve_database_option( + 'hostname', data_source, connection_params, restore=True ) - port = str( - connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), - ) - tls = data_source.get('restore_tls', data_source.get('tls')) + port = utils.resolve_database_option('port', data_source, connection_params, restore=True) + tls = utils.resolve_database_option('tls', data_source, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) - ), + utils.resolve_database_option('username', data_source, connection_params, restore=True), config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['password'] - or data_source.get('restore_password', data_source.get('password')) - ), + utils.resolve_database_option('password', data_source, connection_params, restore=True), config, ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index f3e41dc4..126428b4 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -32,22 +32,15 @@ def make_environment(database, config, restore_connection_params=None): ''' environment = dict(os.environ) - try: - if restore_connection_params: - environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( - ( - restore_connection_params.get('password') - or database.get('restore_password', database['password']) - ), - config, - ) - else: - environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( - database['password'], - config, - ) - except (AttributeError, KeyError): - pass + password = utils.resolve_database_option( + 'password', database, restore_connection_params, restore=restore_connection_params + ) + + if password: + environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( + password, + config, + ) if 'ssl_mode' in database: environment['PGSSLMODE'] = database['ssl_mode'] @@ -91,7 +84,7 @@ def database_names_to_dump(database, config, environment, dry_run): psql_command = tuple( shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql') ) - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) list_command = ( psql_command + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') @@ -196,7 +189,7 @@ def dump_data_sources( ) continue - hostname = utils.get_hostname_from_config(database) + hostname = utils.resolve_database_option('hostname', database) command = ( dump_command + ( @@ -342,18 +335,12 @@ def restore_data_source_dump( hostname, port, username, and password. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' - hostname = connection_params['hostname'] or data_source.get( - 'restore_hostname', - utils.get_hostname_from_config(data_source), - ) - port = str( - connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')), + hostname = utils.resolve_database_option( + 'hostname', data_source, connection_params, restore=True ) + port = utils.resolve_database_option('port', data_source, connection_params, restore=True) username = borgmatic.hooks.credential.parse.resolve_credential( - ( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) - ), + utils.resolve_database_option('username', data_source, connection_params, restore=True), config, ) @@ -372,7 +359,7 @@ def restore_data_source_dump( psql_command + ('--no-password', '--no-psqlrc', '--quiet') + (('--host', hostname) if hostname else ()) - + (('--port', port) if port else ()) + + (('--port', str(port)) if port else ()) + (('--username', username) if username else ()) + (('--dbname', data_source['name']) if not all_databases else ()) + ( @@ -393,7 +380,7 @@ def restore_data_source_dump( + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean')) + (('--dbname', data_source['name']) if not all_databases else ()) + (('--host', hostname) if hostname else ()) - + (('--port', port) if port else ()) + + (('--port', str(port)) if port else ()) + (('--username', username) if username else ()) + (('--no-owner',) if data_source.get('no_owner', False) else ()) + ( diff --git a/borgmatic/hooks/data_source/utils.py b/borgmatic/hooks/data_source/utils.py index 5709c228..1cd56312 100644 --- a/borgmatic/hooks/data_source/utils.py +++ b/borgmatic/hooks/data_source/utils.py @@ -7,10 +7,34 @@ from borgmatic.execute import execute_command_and_capture_output IS_A_HOOK = False -def get_hostname_from_config(database): - if 'container' in database: - return get_ip_from_container(database['container']) - return database.get('hostname', '') +def resolve_database_option(option, data_source, connection_params=None, restore=False): + # Special case `hostname` since it overlaps with `container` + if option == 'hostname': + return _get_hostname_from_config(data_source, connection_params, restore) + if connection_params and (value := connection_params.get(option)): + return value + if restore and f'restore_{option}' in data_source: + return data_source[f'restore_{option}'] + return data_source.get(option) + + +def _get_hostname_from_config(data_source, connection_params=None, restore=False): + # connection params win, full stop + if connection_params: + if container := connection_params.get('container'): + return container + if hostname := connection_params.get('hostname'): + return hostname + # ... then try the restore config + if restore: + if 'restore_container' in data_source: + return get_ip_from_container(data_source['restore_container']) + if 'restore_hostname' in data_source: + return data_source['restore_hostname'] + # ... and finally fall back to the normal options + if 'container' in data_source: + return get_ip_from_container(data_source['container']) + return data_source.get('hostname', '') def get_ip_from_container(container):