From 55c0ab16102d50396af0f0b510dc3307bcc2f4f9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Mar 2025 10:07:03 -0800 Subject: [PATCH] Add "tls" options to the MariaDB and MySQL database hooks. --- NEWS | 2 + borgmatic/config/schema.yaml | 28 +++ borgmatic/hooks/data_source/mariadb.py | 7 + borgmatic/hooks/data_source/mysql.py | 7 + tests/unit/hooks/data_source/test_mariadb.py | 229 ++++++++++++++++++ tests/unit/hooks/data_source/test_mysql.py | 241 +++++++++++++++++++ 6 files changed, 514 insertions(+) diff --git a/NEWS b/NEWS index b6fe702e..c9249dff 100644 --- a/NEWS +++ b/NEWS @@ -8,6 +8,8 @@ "--password" on the command-line! * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit. * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. + * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption + between client and server. 1.9.12 * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 23db735d..de8dec6f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1210,6 +1210,20 @@ properties: Defaults to the "password" option. Supports the "{credential ...}" syntax. example: trustsome1 + tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and server. The default varies based on the + MariaDB version. + example: false + restore_tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and restore server. The default varies based on + the MariaDB version. + example: false mariadb_dump_command: type: string description: | @@ -1340,6 +1354,20 @@ properties: Defaults to the "password" option. Supports the "{credential ...}" syntax. example: trustsome1 + tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and server. The default varies based on the + MySQL installation. + example: false + restore_tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and restore server. The default varies based on + the MySQL installation. + example: false mysql_dump_command: type: string description: | diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index fc879142..cb05f5b9 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -125,6 +125,8 @@ def database_names_to_dump(database, config, username, password, environment, dr + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) @@ -188,6 +190,8 @@ def execute_dump_command( + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -357,6 +361,7 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) + tls = data_source.get('restore_tls', data_source.get('tls')) username = borgmatic.hooks.credential.parse.resolve_credential( ( connection_params['username'] @@ -384,6 +389,8 @@ def restore_data_source_dump( + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) + + (('--ssl',) if tls is True else ()) + + (('--skip-ssl',) if tls is False else ()) ) environment = dict(os.environ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 13d77041..7c5f84f2 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -54,6 +54,8 @@ def database_names_to_dump(database, config, username, password, environment, dr + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) @@ -117,6 +119,8 @@ def execute_dump_command( + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -286,6 +290,7 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) + tls = data_source.get('restore_tls', data_source.get('tls')) username = borgmatic.hooks.credential.parse.resolve_credential( ( connection_params['username'] @@ -317,6 +322,8 @@ def restore_data_source_dump( + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) + + (('--ssl',) if tls is True else ()) + + (('--skip-ssl',) if tls is False else ()) ) environment = dict(os.environ) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index fe183865..c6b88099 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -151,6 +151,64 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): assert names == ('foo', 'bar') +def test_database_names_to_dump_runs_mariadb_with_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mariadb_without_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + def test_use_streaming_true_for_any_databases(): assert module.use_streaming( databases=[flexmock(), flexmock()], @@ -496,6 +554,94 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): ) +def test_execute_dump_command_runs_mariadb_dump_with_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': True}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_without_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--skip-ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': False}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -890,6 +1036,86 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): ) +def test_restore_data_source_dump_runs_mariadb_with_tls(): + hook_config = [{'name': 'foo', 'tls': True}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mariadb_without_tls(): + hook_config = [{'name': 'foo', 'tls': False}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--skip-ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) @@ -990,10 +1216,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'password': 'trustsome1', 'hostname': 'dbhost', 'port': 'dbport', + 'tls': True, 'restore_username': 'restoreuser', 'restore_password': 'restorepass', 'restore_hostname': 'restorehost', 'restore_port': 'restoreport', + 'restore_tls': False, } ] extract_process = flexmock(stdout=flexmock()) @@ -1017,6 +1245,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', + '--skip-ssl', ), processes=[extract_process], output_log_level=logging.DEBUG, diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 2e629dc3..63bb7a72 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -60,6 +60,68 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): assert names == ('foo', 'bar') +def test_database_names_to_dump_runs_mysql_with_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mysql_without_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + def test_use_streaming_true_for_any_databases(): assert module.use_streaming( databases=[flexmock(), flexmock()], @@ -408,6 +470,98 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): ) +def test_execute_dump_command_runs_mysqldump_with_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': True}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mysqldump_without_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--skip-ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': False}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_runs_mysqldump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -816,6 +970,90 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): ) +def test_restore_data_source_dump_runs_mysql_with_tls(): + hook_config = [{'name': 'foo', 'tls': True}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mysql_without_tls(): + hook_config = [{'name': 'foo', 'tls': False}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--skip-ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_runs_mysql_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) @@ -922,10 +1160,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'password': 'trustsome1', 'hostname': 'dbhost', 'port': 'dbport', + 'tls': True, 'restore_username': 'restoreuser', 'restore_password': 'restorepass', 'restore_hostname': 'restorehost', 'restore_port': 'restoreport', + 'restore_tls': False, } ] extract_process = flexmock(stdout=flexmock()) @@ -953,6 +1193,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', + '--skip-ssl', ), processes=[extract_process], output_log_level=logging.DEBUG,