peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-22 02:03:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add "tls" options to the MariaDB and MySQL database hooks.

Browse Source
This commit is contained in:
Dan Helfman
2025-03-03 10:07:03 -08:00
parent 1f86100f26
commit 55c0ab1610
6 changed files with 514 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS
+2
View File
@@ -8,6 +8,8 @@
"--password" on the command-line!
* #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit.
* Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification.
* Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption
between client and server.
1.9.12
* #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will
borgmatic/config/schema.yaml
+28
View File
@@ -1210,6 +1210,20 @@ properties:
Defaults to the "password" option. Supports the
"{credential ...}" syntax.
example: trustsome1
tls:
type: boolean
description: |
Whether to TLS-encrypt data transmitted between the
client and server. The default varies based on the
MariaDB version.
example: false
restore_tls:
type: boolean
description: |
Whether to TLS-encrypt data transmitted between the
client and restore server. The default varies based on
the MariaDB version.
example: false
mariadb_dump_command:
type: string
description: |
@@ -1340,6 +1354,20 @@ properties:
Defaults to the "password" option. Supports the
"{credential ...}" syntax.
example: trustsome1
tls:
type: boolean
description: |
Whether to TLS-encrypt data transmitted between the
client and server. The default varies based on the
MySQL installation.
example: false
restore_tls:
type: boolean
description: |
Whether to TLS-encrypt data transmitted between the
client and restore server. The default varies based on
the MySQL installation.
example: false
mysql_dump_command:
type: string
description: |
borgmatic/hooks/data_source/mariadb.py
+7
View File
@@ -125,6 +125,8 @@ def database_names_to_dump(database, config, username, password, environment, dr
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--skip-column-names', '--batch')
+ ('--execute', 'show schemas')
)
@@ -188,6 +190,8 @@ def execute_dump_command(
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--databases',)
+ database_names
+ ('--result-file', dump_filename)
@@ -357,6 +361,7 @@ def restore_data_source_dump(
port = str(
connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
)
tls = data_source.get('restore_tls', data_source.get('tls'))
username = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['username']
@@ -384,6 +389,8 @@ def restore_data_source_dump(
+ (('--host', hostname) if hostname else ())
+ (('--port', str(port)) if port else ())
+ (('--protocol', 'tcp') if hostname or port else ())
+ (('--ssl',) if tls is True else ())
+ (('--skip-ssl',) if tls is False else ())
)
environment = dict(os.environ)
borgmatic/hooks/data_source/mysql.py
+7
View File
@@ -54,6 +54,8 @@ def database_names_to_dump(database, config, username, password, environment, dr
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--skip-column-names', '--batch')
+ ('--execute', 'show schemas')
)
@@ -117,6 +119,8 @@ def execute_dump_command(
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--databases',)
+ database_names
+ ('--result-file', dump_filename)
@@ -286,6 +290,7 @@ def restore_data_source_dump(
port = str(
connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
)
tls = data_source.get('restore_tls', data_source.get('tls'))
username = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['username']
@@ -317,6 +322,8 @@ def restore_data_source_dump(
+ (('--host', hostname) if hostname else ())
+ (('--port', str(port)) if port else ())
+ (('--protocol', 'tcp') if hostname or port else ())
+ (('--ssl',) if tls is True else ())
+ (('--skip-ssl',) if tls is False else ())
)
environment = dict(os.environ)
tests/unit/hooks/data_source/test_mariadb.py
+229
View File
@@ -151,6 +151,64 @@ def test_database_names_to_dump_queries_mariadb_for_database_names():
assert names == ('foo', 'bar')
def test_database_names_to_dump_runs_mariadb_with_tls():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').with_args(
'root', 'trustsome1', None
).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'mariadb',
'--defaults-extra-file=/dev/fd/99',
'--ssl',
'--skip-column-names',
'--batch',
'--execute',
'show schemas',
),
environment=environment,
).and_return('foo\nbar\nmysql\n').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False
)
assert names == ('foo', 'bar')
def test_database_names_to_dump_runs_mariadb_without_tls():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').with_args(
'root', 'trustsome1', None
).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'mariadb',
'--defaults-extra-file=/dev/fd/99',
'--skip-ssl',
'--skip-column-names',
'--batch',
'--execute',
'show schemas',
),
environment=environment,
).and_return('foo\nbar\nmysql\n').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False
)
assert names == ('foo', 'bar')
def test_use_streaming_true_for_any_databases():
assert module.use_streaming(
databases=[flexmock(), flexmock()],
@@ -496,6 +554,94 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port():
)
def test_execute_dump_command_runs_mariadb_dump_with_tls():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').with_args(
'root', 'trustsome1', None
).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--defaults-extra-file=/dev/fd/99',
'--add-drop-database',
'--ssl',
'--databases',
'foo',
'--result-file',
'dump',
),
environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'tls': True},
config={},
username='root',
password='trustsome1',
dump_path=flexmock(),
database_names=('foo',),
environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mariadb_dump_without_tls():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').with_args(
'root', 'trustsome1', None
).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--defaults-extra-file=/dev/fd/99',
'--add-drop-database',
'--skip-ssl',
'--databases',
'foo',
'--result-file',
'dump',
),
environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'tls': False},
config={},
username='root',
password='trustsome1',
dump_path=flexmock(),
database_names=('foo',),
environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mariadb_dump_with_username_and_password():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -890,6 +1036,86 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port():
)
def test_restore_data_source_dump_runs_mariadb_with_tls():
hook_config = [{'name': 'foo', 'tls': True}]
extract_process = flexmock(stdout=flexmock())
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').with_args(
None, None, None
).and_return(())
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mariadb',
'--batch',
'--ssl',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment={'USER': 'root'},
).once()
module.restore_data_source_dump(
hook_config,
{},
data_source=hook_config[0],
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
borgmatic_runtime_directory='/run/borgmatic',
)
def test_restore_data_source_dump_runs_mariadb_without_tls():
hook_config = [{'name': 'foo', 'tls': False}]
extract_process = flexmock(stdout=flexmock())
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').with_args(
None, None, None
).and_return(())
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mariadb',
'--batch',
'--skip-ssl',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment={'USER': 'root'},
).once()
module.restore_data_source_dump(
hook_config,
{},
data_source=hook_config[0],
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
borgmatic_runtime_directory='/run/borgmatic',
)
def test_restore_data_source_dump_runs_mariadb_with_username_and_password():
hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
extract_process = flexmock(stdout=flexmock())
@@ -990,10 +1216,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
'password': 'trustsome1',
'hostname': 'dbhost',
'port': 'dbport',
'tls': True,
'restore_username': 'restoreuser',
'restore_password': 'restorepass',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_tls': False,
}
]
extract_process = flexmock(stdout=flexmock())
@@ -1017,6 +1245,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
'restoreport',
'--protocol',
'tcp',
'--skip-ssl',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
tests/unit/hooks/data_source/test_mysql.py
+241
View File
@@ -60,6 +60,68 @@ def test_database_names_to_dump_queries_mysql_for_database_names():
assert names == ('foo', 'bar')
def test_database_names_to_dump_runs_mysql_with_tls():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'mysql',
'--defaults-extra-file=/dev/fd/99',
'--ssl',
'--skip-column-names',
'--batch',
'--execute',
'show schemas',
),
environment=environment,
).and_return('foo\nbar\nmysql\n').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False
)
assert names == ('foo', 'bar')
def test_database_names_to_dump_runs_mysql_without_tls():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'mysql',
'--defaults-extra-file=/dev/fd/99',
'--skip-ssl',
'--skip-column-names',
'--batch',
'--execute',
'show schemas',
),
environment=environment,
).and_return('foo\nbar\nmysql\n').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False
)
assert names == ('foo', 'bar')
def test_use_streaming_true_for_any_databases():
assert module.use_streaming(
databases=[flexmock(), flexmock()],
@@ -408,6 +470,98 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port():
)
def test_execute_dump_command_runs_mysqldump_with_tls():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mysqldump',
'--defaults-extra-file=/dev/fd/99',
'--add-drop-database',
'--ssl',
'--databases',
'foo',
'--result-file',
'dump',
),
environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'tls': True},
config={},
username='root',
password='trustsome1',
dump_path=flexmock(),
database_names=('foo',),
environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mysqldump_without_tls():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mysqldump',
'--defaults-extra-file=/dev/fd/99',
'--add-drop-database',
'--skip-ssl',
'--databases',
'foo',
'--result-file',
'dump',
),
environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'tls': False},
config={},
username='root',
password='trustsome1',
dump_path=flexmock(),
database_names=('foo',),
environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mysqldump_with_username_and_password():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -816,6 +970,90 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port():
)
def test_restore_data_source_dump_runs_mysql_with_tls():
hook_config = [{'name': 'foo', 'tls': True}]
extract_process = flexmock(stdout=flexmock())
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).with_args(None, None, None).and_return(())
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mysql',
'--batch',
'--ssl',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment={'USER': 'root'},
).once()
module.restore_data_source_dump(
hook_config,
{},
data_source=hook_config[0],
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
borgmatic_runtime_directory='/run/borgmatic',
)
def test_restore_data_source_dump_runs_mysql_without_tls():
hook_config = [{'name': 'foo', 'tls': False}]
extract_process = flexmock(stdout=flexmock())
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).with_args(None, None, None).and_return(())
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'mysql',
'--batch',
'--skip-ssl',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment={'USER': 'root'},
).once()
module.restore_data_source_dump(
hook_config,
{},
data_source=hook_config[0],
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
borgmatic_runtime_directory='/run/borgmatic',
)
def test_restore_data_source_dump_runs_mysql_with_username_and_password():
hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
extract_process = flexmock(stdout=flexmock())
@@ -922,10 +1160,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
'password': 'trustsome1',
'hostname': 'dbhost',
'port': 'dbport',
'tls': True,
'restore_username': 'restoreuser',
'restore_password': 'restorepass',
'restore_hostname': 'restorehost',
'restore_port': 'restoreport',
'restore_tls': False,
}
]
extract_process = flexmock(stdout=flexmock())
@@ -953,6 +1193,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
'restoreport',
'--protocol',
'tcp',
'--skip-ssl',
),
processes=[extract_process],
output_log_level=logging.DEBUG,