From 091e9fa9ee4066f52ef27c818448382ef3b3a72e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 29 Apr 2025 14:19:15 -0700 Subject: [PATCH] Add a "password_transport" option to the MariaDB and MySQL database hooks for customizing how borgmatic transmits passwords to the database client (#1033). --- NEWS | 10 +- borgmatic/config/schema.yaml | 34 +++ borgmatic/hooks/data_source/mariadb.py | 38 ++- borgmatic/hooks/data_source/mysql.py | 44 ++- docs/how-to/backup-your-databases.md | 34 ++- tests/unit/hooks/data_source/test_mariadb.py | 287 ++++++++++++++---- tests/unit/hooks/data_source/test_mysql.py | 299 +++++++++++++++---- 7 files changed, 620 insertions(+), 126 deletions(-) diff --git a/NEWS b/NEWS index 57885c60..c6ababa8 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 2.0.5.dev0 + * #1033: Add a "password_transport" option to the MariaDB and MySQL database hooks for customizing + how borgmatic transmits passwords to the database client. * #1078: Add "keep_3monthly" and "keep_13weekly" options for customizing "prune" action archive retention. * #1078: Add a "use_chunks_archive" option for controlling whether Borg uses its chunks cache @@ -76,7 +78,7 @@ * #837: Add custom command options for the MongoDB hook. * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: - https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers * #1037: Fix an error with the "extract" action when both a remote repository and a "working_directory" are used. * #1044: Fix an error in the systemd credential hook when the credential name contains a "." @@ -326,7 +328,7 @@ paths when a "working_directory" is set. * #906: Add documentation details for how to run custom database dump commands using binaries from running containers: - https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers * Fix a regression in which the "color" option had no effect. * Add a recent contributors section to the documentation, because credit where credit's due! See: https://torsion.org/borgmatic/#recent-contributors @@ -382,7 +384,7 @@ * Fix handling of the NO_COLOR environment variable to ignore an empty value. * Add documentation about backing up containerized databases by configuring borgmatic to exec into a container to run a dump command: - https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers 1.8.9 * #311: Add custom dump/restore command options for MySQL and MariaDB. @@ -652,7 +654,7 @@ at the command-line. See the configuration reference for more information: https://torsion.org/borgmatic/docs/reference/configuration/ * #649: Add documentation on backing up a database running in a container: - https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers * #655: Fix error when databases are configured and a source directory doesn't exist. * Add code style plugins to enforce use of Python f-strings and prevent single-letter variables. To join in the pedantry, refresh your test environment with "tox --recreate". diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 494b0d70..a2cdbca8 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1560,6 +1560,23 @@ properties: Defaults to the "password" option. Supports the "{credential ...}" syntax. example: trustsome1 + password_transport: + type: string + enum: + - pipe + - environment + description: | + How to transmit database passwords from borgmatic to the + MariaDB client, one of: + * "pipe": Securely transmit passwords via anonymous + pipe. Only works if the database client is on the + same host as borgmatic. (The server can be + somewhere else.) This is the default value. + * "environment": Transmit passwords via environment + variable. Potentially less secure than a pipe, but + necessary when the database client is elsewhere, e.g. + when "mariadb_dump_command" is configured to "exec" + into a container and run a client there. tls: type: boolean description: | @@ -1708,6 +1725,23 @@ properties: Defaults to the "password" option. Supports the "{credential ...}" syntax. example: trustsome1 + password_transport: + type: string + enum: + - pipe + - environment + description: | + How to transmit database passwords from borgmatic to the + MySQL client, one of: + * "pipe": Securely transmit passwords via anonymous + pipe. Only works if the database client is on the + same host as borgmatic. (The server can be + somewhere else.) This is the default value. + * "environment": Transmit passwords via environment + variable. Potentially less secure than a pipe, but + necessary when the database client is elsewhere, e.g. + when "mysql_dump_command" is configured to "exec" + into a container and run a client there. tls: type: boolean description: | diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 1f140793..286b5aef 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -120,13 +120,19 @@ def database_names_to_dump(database, config, username, password, environment, dr shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb') ) extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) + password_transport = database.get('password_transport', 'pipe') show_command = ( mariadb_show_command - + make_defaults_file_options(username, password, defaults_extra_filename) + + ( + make_defaults_file_options(username, password, defaults_extra_filename) + if password_transport == 'pipe' + else () + ) + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') @@ -184,14 +190,20 @@ def execute_dump_command( for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') ) extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) + password_transport = database.get('password_transport', 'pipe') dump_command = ( mariadb_dump_command - + make_defaults_file_options(username, password, defaults_extra_filename) + + ( + make_defaults_file_options(username, password, defaults_extra_filename) + if password_transport == 'pipe' + else () + ) + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) @@ -255,7 +267,14 @@ def dump_data_sources( password = borgmatic.hooks.credential.parse.resolve_credential( database.get('password'), config ) - environment = dict(os.environ) + environment = dict( + os.environ, + **( + {'MYSQL_PWD': password} + if password and database.get('password_transport') == 'environment' + else {} + ), + ) dump_database_names = database_names_to_dump( database, config, username, password, environment, dry_run ) @@ -383,18 +402,27 @@ def restore_data_source_dump( shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb') ) extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options')) + password_transport = data_source.get('password_transport', 'pipe') restore_command = ( mariadb_restore_command - + make_defaults_file_options(username, password, defaults_extra_filename) + + ( + make_defaults_file_options(username, password, defaults_extra_filename) + if password_transport == 'pipe' + else () + ) + extra_options + ('--batch',) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) + + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if tls is True else ()) + (('--skip-ssl',) if tls is False else ()) ) - environment = dict(os.environ) + environment = dict( + os.environ, + **({'MYSQL_PWD': password} if password and password_transport == 'environment' else {}), + ) logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 7c5f84f2..2a1b6db9 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -45,15 +45,21 @@ def database_names_to_dump(database, config, username, password, environment, dr extra_options, defaults_extra_filename = ( borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) ) + password_transport = database.get('password_transport', 'pipe') show_command = ( mysql_show_command - + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( - username, password, defaults_extra_filename + + ( + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename + ) + if password_transport == 'pipe' + else () ) + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') @@ -109,16 +115,22 @@ def execute_dump_command( extra_options, defaults_extra_filename = ( borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) ) + password_transport = database.get('password_transport', 'pipe') dump_command = ( mysql_dump_command - + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( - username, password, defaults_extra_filename + + ( + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename + ) + if password_transport == 'pipe' + else () ) + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) @@ -182,7 +194,14 @@ def dump_data_sources( password = borgmatic.hooks.credential.parse.resolve_credential( database.get('password'), config ) - environment = dict(os.environ) + environment = dict( + os.environ, + **( + {'MYSQL_PWD': password} + if password and database.get('password_transport') == 'environment' + else {} + ), + ) dump_database_names = database_names_to_dump( database, config, username, password, environment, dry_run ) @@ -312,20 +331,29 @@ def restore_data_source_dump( extra_options, defaults_extra_filename = ( borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options')) ) + password_transport = data_source.get('password_transport', 'pipe') restore_command = ( mysql_restore_command - + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( - username, password, defaults_extra_filename + + ( + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename + ) + if password_transport == 'pipe' + else () ) + extra_options + ('--batch',) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) + + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if tls is True else ()) + (('--skip-ssl',) if tls is False else ()) ) - environment = dict(os.environ) + environment = dict( + os.environ, + **({'MYSQL_PWD': password} if password and password_transport == 'environment' else {}), + ) logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 08e6bb10..3088a39b 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -191,7 +191,7 @@ mysql_databases: format: sql ``` -### Containers +### Database containers If your database server is running within a container and borgmatic is too, no problem—configure borgmatic to connect to the container's name on its exposed @@ -209,6 +209,9 @@ postgresql_databases: Prior to version 1.8.0 Put these options in the `hooks:` section of your configuration. + +#### Database client on the host + But what if borgmatic is running on the host? You can still connect to a database server container if its ports are properly exposed to the host. For instance, when running the database container, you can specify `--publish @@ -239,6 +242,9 @@ hooks: Alter the ports in these examples to suit your particular database system. + +#### Database client in a running container + Normally, borgmatic dumps a database by running a database dump command (e.g. `pg_dump`) on the host or wherever borgmatic is running, and this command connects to your containerized database via the given `hostname` and `port`. But @@ -282,6 +288,32 @@ services: - /run/user/1000:/run/user/1000 ``` +And here's an example of using a MariaDB database client within a running Docker +container: + +```yaml +hooks: + mariadb_databases: + - name: users + hostname: 127.0.0.1 + username: example + password: trustsome1 + password_transport: environment + mariadb_dump_command: docker exec --env MYSQL_PWD my_mariadb_container mariadb-dump +``` + +The `password_transport: environment` option tells borgmatic to transmit the +password via environment variable instead of the default behavior of using an +anonymous pipe. The environment variable transport is potentially less secure +than the pipe, but it's necessary when the database client is running in a +container separate from borgmatic. + +A similar approach can work with MySQL, using `mysql_dump_command` instead of +`mariadb_dump_command` to run `mysqldump` in a container. + + +#### Database client in a temporary container + Another variation: If you're running borgmatic on the host but want to spin up a temporary `pg_dump` container whenever borgmatic dumps a database, for instance to make use of a `pg_dump` version not present on the host, try diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 871c1969..21a1ff7e 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -167,6 +167,38 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): assert names == ('foo', 'bar') +def test_database_names_to_dump_with_environment_password_transport_skips_defaults_file_and_passes_user_flag(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').never() + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--user', + 'root', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'password_transport': 'environment'}, + {}, + 'root', + 'trustsome1', + environment, + dry_run=False, + ) + + assert names == ('foo', 'bar') + + def test_database_names_to_dump_runs_mariadb_with_tls(): environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( @@ -247,9 +279,22 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( - ('bar',) - ) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=databases[0], + config={}, + username=None, + password=None, + environment={'USER': 'root'}, + dry_run=False, + ).and_return(('foo',)) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=databases[1], + config={}, + username=None, + password=None, + environment={'USER': 'root'}, + dry_run=False, + ).and_return(('bar',)) for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( @@ -285,9 +330,14 @@ def test_dump_data_sources_dumps_with_password(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( - ('bar',) - ) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=database, + config={}, + username='root', + password='trustsome1', + environment={'USER': 'root'}, + dry_run=False, + ).and_return(('foo',)) flexmock(module).should_receive('execute_dump_command').with_args( database=database, @@ -311,6 +361,50 @@ def test_dump_data_sources_dumps_with_password(): ) == [process] +def test_dump_data_sources_dumps_with_environment_password_transport_passes_password_environment_variable(): + database = { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'password_transport': 'environment', + } + process = flexmock() + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=database, + config={}, + username='root', + password='trustsome1', + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + dry_run=False, + ).and_return(('foo',)) + + flexmock(module).should_receive('execute_dump_command').with_args( + database=database, + config={}, + username='root', + password='trustsome1', + dump_path=object, + database_names=('foo',), + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + dry_run=object, + dry_run_label=object, + ).and_return(process).once() + + assert module.dump_data_sources( + [database], + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) == [process] + + def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() @@ -378,6 +472,54 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured ) +def test_dump_data_sources_errors_for_missing_all_databases(): + databases = [{'name': 'all'}] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/all' + ) + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + with pytest.raises(ValueError): + assert module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + + +def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): + databases = [{'name': 'all'}] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/all' + ) + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=True, + ) + == [] + ) + + def test_database_names_to_dump_runs_mariadb_with_list_options(): database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl'} flexmock(module).should_receive('parse_extra_options').and_return( @@ -479,6 +621,48 @@ def test_execute_dump_command_runs_mariadb_dump(): ) +def test_execute_dump_command_with_environment_password_transport_skips_defaults_file_and_passes_user_flag(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').never() + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--add-drop-database', + '--user', + 'root', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'password_transport': 'environment'}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -849,54 +1033,6 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): ) -def test_dump_data_sources_errors_for_missing_all_databases(): - databases = [{'name': 'all'}] - flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.hooks.credential.parse).should_receive( - 'resolve_credential' - ).replace_with(lambda value, config: value) - flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( - 'databases/localhost/all' - ) - flexmock(module).should_receive('database_names_to_dump').and_return(()) - - with pytest.raises(ValueError): - assert module.dump_data_sources( - databases, - {}, - config_paths=('test.yaml',), - borgmatic_runtime_directory='/run/borgmatic', - patterns=[], - dry_run=False, - ) - - -def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): - databases = [{'name': 'all'}] - flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.hooks.credential.parse).should_receive( - 'resolve_credential' - ).replace_with(lambda value, config: value) - flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( - 'databases/localhost/all' - ) - flexmock(module).should_receive('database_names_to_dump').and_return(()) - - assert ( - module.dump_data_sources( - databases, - {}, - config_paths=('test.yaml',), - borgmatic_runtime_directory='/run/borgmatic', - patterns=[], - dry_run=True, - ) - == [] - ) - - def test_restore_data_source_dump_runs_mariadb_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) @@ -1168,6 +1304,49 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): ) +def test_restore_data_source_with_environment_password_transport_skips_defaults_file_and_passes_user_flag(): + hook_config = [ + { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'password_transport': 'environment', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').never() + flexmock(module.os).should_receive('environ').and_return( + {'USER': 'root', 'MYSQL_PWD': 'trustsome1'} + ) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('mariadb', '--batch', '--user', 'root'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore(): hook_config = [ { diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 63bb7a72..4eac5cb7 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -60,6 +60,42 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): assert names == ('foo', 'bar') +def test_database_names_to_dump_with_environment_password_transport_skips_defaults_file_and_passes_user_flag(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).never() + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mysql', + '--user', + 'root', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'password_transport': 'environment'}, + {}, + 'root', + 'trustsome1', + environment, + dry_run=False, + ) + + assert names == ('foo', 'bar') + + def test_database_names_to_dump_runs_mysql_with_tls(): environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( @@ -141,9 +177,22 @@ def test_dump_data_sources_dumps_each_database(): 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( - ('bar',) - ) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=databases[0], + config={}, + username=None, + password=None, + environment={'USER': 'root'}, + dry_run=False, + ).and_return(('foo',)) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=databases[1], + config={}, + username=None, + password=None, + environment={'USER': 'root'}, + dry_run=False, + ).and_return(('bar',)) for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( @@ -179,9 +228,14 @@ def test_dump_data_sources_dumps_with_password(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( - ('bar',) - ) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=database, + config={}, + username='root', + password='trustsome1', + environment={'USER': 'root'}, + dry_run=False, + ).and_return(('foo',)) flexmock(module).should_receive('execute_dump_command').with_args( database=database, @@ -205,6 +259,50 @@ def test_dump_data_sources_dumps_with_password(): ) == [process] +def test_dump_data_sources_dumps_with_environment_password_transport_passes_password_environment_variable(): + database = { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'password_transport': 'environment', + } + process = flexmock() + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('database_names_to_dump').with_args( + database=database, + config={}, + username='root', + password='trustsome1', + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + dry_run=False, + ).and_return(('foo',)) + + flexmock(module).should_receive('execute_dump_command').with_args( + database=database, + config={}, + username='root', + password='trustsome1', + dump_path=object, + database_names=('foo',), + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + dry_run=object, + dry_run_label=object, + ).and_return(process).once() + + assert module.dump_data_sources( + [database], + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) == [process] + + def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() @@ -272,6 +370,54 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured ) +def test_dump_data_sources_errors_for_missing_all_databases(): + databases = [{'name': 'all'}] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/all' + ) + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + with pytest.raises(ValueError): + assert module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + + +def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): + databases = [{'name': 'all'}] + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/all' + ) + flexmock(module).should_receive('database_names_to_dump').and_return(()) + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=True, + ) + == [] + ) + + def test_database_names_to_dump_runs_mysql_with_list_options(): database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf --skip-ssl'} flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( @@ -375,6 +521,52 @@ def test_execute_dump_command_runs_mysqldump(): ) +def test_execute_dump_command_with_environment_password_transport_skips_defaults_file_and_passes_user_flag(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).never() + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--add-drop-database', + '--user', + 'root', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'password_transport': 'environment'}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -761,54 +953,6 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): ) -def test_dump_data_sources_errors_for_missing_all_databases(): - databases = [{'name': 'all'}] - flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.hooks.credential.parse).should_receive( - 'resolve_credential' - ).replace_with(lambda value, config: value) - flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( - 'databases/localhost/all' - ) - flexmock(module).should_receive('database_names_to_dump').and_return(()) - - with pytest.raises(ValueError): - assert module.dump_data_sources( - databases, - {}, - config_paths=('test.yaml',), - borgmatic_runtime_directory='/run/borgmatic', - patterns=[], - dry_run=False, - ) - - -def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): - databases = [{'name': 'all'}] - flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.hooks.credential.parse).should_receive( - 'resolve_credential' - ).replace_with(lambda value, config: value) - flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( - 'databases/localhost/all' - ) - flexmock(module).should_receive('database_names_to_dump').and_return(()) - - assert ( - module.dump_data_sources( - databases, - {}, - config_paths=('test.yaml',), - borgmatic_runtime_directory='/run/borgmatic', - patterns=[], - dry_run=True, - ) - == [] - ) - - def test_restore_data_source_dump_runs_mysql_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) @@ -1092,6 +1236,53 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): ) +def test_restore_data_source_with_environment_password_transport_skips_defaults_file_and_passes_user_flag(): + hook_config = [ + { + 'name': 'foo', + 'username': 'root', + 'password': 'trustsome1', + 'password_transport': 'environment', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).never() + flexmock(module.os).should_receive('environ').and_return( + {'USER': 'root', 'MYSQL_PWD': 'trustsome1'} + ) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ('mysql', '--batch', '--user', 'root'), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore(): hook_config = [ {