peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-31 21:53:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add a "password_transport" option to the MariaDB and MySQL database hooks for customizing how borgmatic transmits passwords to the database client (#1033).

Browse Source
This commit is contained in:
Dan Helfman
2025-04-29 14:19:15 -07:00
parent 0f4e93c9c8
commit 091e9fa9ee
7 changed files with 620 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS
+6 -4
View File
@@ -1,4 +1,6 @@
2.0.5.dev0
* #1033: Add a "password_transport" option to the MariaDB and MySQL database hooks for customizing
how borgmatic transmits passwords to the database client.
* #1078: Add "keep_3monthly" and "keep_13weekly" options for customizing "prune" action archive
retention.
* #1078: Add a "use_chunks_archive" option for controlling whether Borg uses its chunks cache
@@ -76,7 +78,7 @@
* #837: Add custom command options for the MongoDB hook.
* #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune".
* #1020: Document a database use case involving a temporary database client container:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
* #1037: Fix an error with the "extract" action when both a remote repository and a
"working_directory" are used.
* #1044: Fix an error in the systemd credential hook when the credential name contains a "."
@@ -326,7 +328,7 @@
paths when a "working_directory" is set.
* #906: Add documentation details for how to run custom database dump commands using binaries from
running containers:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
* Fix a regression in which the "color" option had no effect.
* Add a recent contributors section to the documentation, because credit where credit's due! See:
https://torsion.org/borgmatic/#recent-contributors
@@ -382,7 +384,7 @@
* Fix handling of the NO_COLOR environment variable to ignore an empty value.
* Add documentation about backing up containerized databases by configuring borgmatic to exec into
a container to run a dump command:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
1.8.9
* #311: Add custom dump/restore command options for MySQL and MariaDB.
@@ -652,7 +654,7 @@
at the command-line. See the configuration reference for more information:
https://torsion.org/borgmatic/docs/reference/configuration/
* #649: Add documentation on backing up a database running in a container:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-containers
* #655: Fix error when databases are configured and a source directory doesn't exist.
* Add code style plugins to enforce use of Python f-strings and prevent single-letter variables.
To join in the pedantry, refresh your test environment with "tox --recreate".
borgmatic/config/schema.yaml
+34
View File
@@ -1560,6 +1560,23 @@ properties:
Defaults to the "password" option. Supports the
"{credential ...}" syntax.
example: trustsome1
password_transport:
type: string
enum:
- pipe
- environment
description: |
How to transmit database passwords from borgmatic to the
MariaDB client, one of:
* "pipe": Securely transmit passwords via anonymous
pipe. Only works if the database client is on the
same host as borgmatic. (The server can be
somewhere else.) This is the default value.
* "environment": Transmit passwords via environment
variable. Potentially less secure than a pipe, but
necessary when the database client is elsewhere, e.g.
when "mariadb_dump_command" is configured to "exec"
into a container and run a client there.
tls:
type: boolean
description: |
@@ -1708,6 +1725,23 @@ properties:
Defaults to the "password" option. Supports the
"{credential ...}" syntax.
example: trustsome1
password_transport:
type: string
enum:
- pipe
- environment
description: |
How to transmit database passwords from borgmatic to the
MySQL client, one of:
* "pipe": Securely transmit passwords via anonymous
pipe. Only works if the database client is on the
same host as borgmatic. (The server can be
somewhere else.) This is the default value.
* "environment": Transmit passwords via environment
variable. Potentially less secure than a pipe, but
necessary when the database client is elsewhere, e.g.
when "mysql_dump_command" is configured to "exec"
into a container and run a client there.
tls:
type: boolean
description: |
borgmatic/hooks/data_source/mariadb.py
+33 -5
View File
@@ -120,13 +120,19 @@ def database_names_to_dump(database, config, username, password, environment, dr
shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb')
)
extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options'))
password_transport = database.get('password_transport', 'pipe')
show_command = (
mariadb_show_command
+ make_defaults_file_options(username, password, defaults_extra_filename)
+ (
make_defaults_file_options(username, password, defaults_extra_filename)
if password_transport == 'pipe'
else ()
)
+ extra_options
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--skip-column-names', '--batch')
@@ -184,14 +190,20 @@ def execute_dump_command(
for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump')
)
extra_options, defaults_extra_filename = parse_extra_options(database.get('options'))
password_transport = database.get('password_transport', 'pipe')
dump_command = (
mariadb_dump_command
+ make_defaults_file_options(username, password, defaults_extra_filename)
+ (
make_defaults_file_options(username, password, defaults_extra_filename)
if password_transport == 'pipe'
else ()
)
+ extra_options
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--databases',)
@@ -255,7 +267,14 @@ def dump_data_sources(
password = borgmatic.hooks.credential.parse.resolve_credential(
database.get('password'), config
)
environment = dict(os.environ)
environment = dict(
os.environ,
**(
{'MYSQL_PWD': password}
if password and database.get('password_transport') == 'environment'
else {}
),
)
dump_database_names = database_names_to_dump(
database, config, username, password, environment, dry_run
)
@@ -383,18 +402,27 @@ def restore_data_source_dump(
shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb')
)
extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options'))
password_transport = data_source.get('password_transport', 'pipe')
restore_command = (
mariadb_restore_command
+ make_defaults_file_options(username, password, defaults_extra_filename)
+ (
make_defaults_file_options(username, password, defaults_extra_filename)
if password_transport == 'pipe'
else ()
)
+ extra_options
+ ('--batch',)
+ (('--host', hostname) if hostname else ())
+ (('--port', str(port)) if port else ())
+ (('--protocol', 'tcp') if hostname or port else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if tls is True else ())
+ (('--skip-ssl',) if tls is False else ())
)
environment = dict(os.environ)
environment = dict(
os.environ,
**({'MYSQL_PWD': password} if password and password_transport == 'environment' else {}),
)
logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}")
if dry_run:
borgmatic/hooks/data_source/mysql.py
+36 -8
View File
@@ -45,15 +45,21 @@ def database_names_to_dump(database, config, username, password, environment, dr
extra_options, defaults_extra_filename = (
borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options'))
)
password_transport = database.get('password_transport', 'pipe')
show_command = (
mysql_show_command
+ borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
username, password, defaults_extra_filename
+ (
borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
username, password, defaults_extra_filename
)
if password_transport == 'pipe'
else ()
)
+ extra_options
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--skip-column-names', '--batch')
@@ -109,16 +115,22 @@ def execute_dump_command(
extra_options, defaults_extra_filename = (
borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options'))
)
password_transport = database.get('password_transport', 'pipe')
dump_command = (
mysql_dump_command
+ borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
username, password, defaults_extra_filename
+ (
borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
username, password, defaults_extra_filename
)
if password_transport == 'pipe'
else ()
)
+ extra_options
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--databases',)
@@ -182,7 +194,14 @@ def dump_data_sources(
password = borgmatic.hooks.credential.parse.resolve_credential(
database.get('password'), config
)
environment = dict(os.environ)
environment = dict(
os.environ,
**(
{'MYSQL_PWD': password}
if password and database.get('password_transport') == 'environment'
else {}
),
)
dump_database_names = database_names_to_dump(
database, config, username, password, environment, dry_run
)
@@ -312,20 +331,29 @@ def restore_data_source_dump(
extra_options, defaults_extra_filename = (
borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options'))
)
password_transport = data_source.get('password_transport', 'pipe')
restore_command = (
mysql_restore_command
+ borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
username, password, defaults_extra_filename
+ (
borgmatic.hooks.data_source.mariadb.make_defaults_file_options(
username, password, defaults_extra_filename
)
if password_transport == 'pipe'
else ()
)
+ extra_options
+ ('--batch',)
+ (('--host', hostname) if hostname else ())
+ (('--port', str(port)) if port else ())
+ (('--protocol', 'tcp') if hostname or port else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if tls is True else ())
+ (('--skip-ssl',) if tls is False else ())
)
environment = dict(os.environ)
environment = dict(
os.environ,
**({'MYSQL_PWD': password} if password and password_transport == 'environment' else {}),
)
logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}")
if dry_run:
docs/how-to/backup-your-databases.md
+33 -1
View File
@@ -191,7 +191,7 @@ mysql_databases:
format: sql
```
### Containers
### Database containers
If your database server is running within a container and borgmatic is too, no
problem—configure borgmatic to connect to the container's name on its exposed
@@ -209,6 +209,9 @@ postgresql_databases:
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
these options in the `hooks:` section of your configuration.
#### Database client on the host
But what if borgmatic is running on the host? You can still connect to a
database server container if its ports are properly exposed to the host. For
instance, when running the database container, you can specify `--publish
@@ -239,6 +242,9 @@ hooks:
Alter the ports in these examples to suit your particular database system.
#### Database client in a running container
Normally, borgmatic dumps a database by running a database dump command (e.g.
`pg_dump`) on the host or wherever borgmatic is running, and this command
connects to your containerized database via the given `hostname` and `port`. But
@@ -282,6 +288,32 @@ services:
- /run/user/1000:/run/user/1000
```
And here's an example of using a MariaDB database client within a running Docker
container:
```yaml
hooks:
mariadb_databases:
- name: users
hostname: 127.0.0.1
username: example
password: trustsome1
password_transport: environment
mariadb_dump_command: docker exec --env MYSQL_PWD my_mariadb_container mariadb-dump
```
The `password_transport: environment` option tells borgmatic to transmit the
password via environment variable instead of the default behavior of using an
anonymous pipe. The environment variable transport is potentially less secure
than the pipe, but it's necessary when the database client is running in a
container separate from borgmatic.
A similar approach can work with MySQL, using `mysql_dump_command` instead of
`mariadb_dump_command` to run `mysqldump` in a container.
#### Database client in a temporary container
Another variation: If you're running borgmatic on the host but want to spin up a
temporary `pg_dump` container whenever borgmatic dumps a database, for
instance to make use of a `pg_dump` version not present on the host, try
tests/unit/hooks/data_source/test_mariadb.py
+233 -54
View File
@@ -167,6 +167,38 @@ def test_database_names_to_dump_queries_mariadb_for_database_names():
assert names == ('foo', 'bar')
def test_database_names_to_dump_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').never()
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'mariadb',
'--user',
'root',
'--skip-column-names',
'--batch',
'--execute',
'show schemas',
),
environment=environment,
).and_return('foo\nbar\nmysql\n').once()
names = module.database_names_to_dump(
{'name': 'all', 'password_transport': 'environment'},
{},
'root',
'trustsome1',
environment,
dry_run=False,
)
assert names == ('foo', 'bar')
def test_database_names_to_dump_runs_mariadb_with_tls():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
@@ -247,9 +279,22 @@ def test_dump_data_sources_dumps_each_database():
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
('bar',)
)
flexmock(module).should_receive('database_names_to_dump').with_args(
database=databases[0],
config={},
username=None,
password=None,
environment={'USER': 'root'},
dry_run=False,
).and_return(('foo',))
flexmock(module).should_receive('database_names_to_dump').with_args(
database=databases[1],
config={},
username=None,
password=None,
environment={'USER': 'root'},
dry_run=False,
).and_return(('bar',))
for name, process in zip(('foo', 'bar'), processes):
flexmock(module).should_receive('execute_dump_command').with_args(
@@ -285,9 +330,14 @@ def test_dump_data_sources_dumps_with_password():
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
('bar',)
)
flexmock(module).should_receive('database_names_to_dump').with_args(
database=database,
config={},
username='root',
password='trustsome1',
environment={'USER': 'root'},
dry_run=False,
).and_return(('foo',))
flexmock(module).should_receive('execute_dump_command').with_args(
database=database,
@@ -311,6 +361,50 @@ def test_dump_data_sources_dumps_with_password():
) == [process]
def test_dump_data_sources_dumps_with_environment_password_transport_passes_password_environment_variable():
database = {
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'password_transport': 'environment',
}
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('database_names_to_dump').with_args(
database=database,
config={},
username='root',
password='trustsome1',
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
dry_run=False,
).and_return(('foo',))
flexmock(module).should_receive('execute_dump_command').with_args(
database=database,
config={},
username='root',
password='trustsome1',
dump_path=object,
database_names=('foo',),
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
dry_run=object,
dry_run_label=object,
).and_return(process).once()
assert module.dump_data_sources(
[database],
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=False,
) == [process]
def test_dump_data_sources_dumps_all_databases_at_once():
databases = [{'name': 'all'}]
process = flexmock()
@@ -378,6 +472,54 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
)
def test_dump_data_sources_errors_for_missing_all_databases():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
with pytest.raises(ValueError):
assert module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=False,
)
def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
assert (
module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=True,
)
== []
)
def test_database_names_to_dump_runs_mariadb_with_list_options():
database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl'}
flexmock(module).should_receive('parse_extra_options').and_return(
@@ -479,6 +621,48 @@ def test_execute_dump_command_runs_mariadb_dump():
)
def test_execute_dump_command_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').never()
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--add-drop-database',
'--user',
'root',
'--databases',
'foo',
'--result-file',
'dump',
),
environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'password_transport': 'environment'},
config={},
username='root',
password='trustsome1',
dump_path=flexmock(),
database_names=('foo',),
environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -849,54 +1033,6 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump():
)
def test_dump_data_sources_errors_for_missing_all_databases():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
with pytest.raises(ValueError):
assert module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=False,
)
def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
assert (
module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=True,
)
== []
)
def test_restore_data_source_dump_runs_mariadb_to_restore():
hook_config = [{'name': 'foo'}, {'name': 'bar'}]
extract_process = flexmock(stdout=flexmock())
@@ -1168,6 +1304,49 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password():
)
def test_restore_data_source_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
hook_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'password_transport': 'environment',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module).should_receive('make_defaults_file_options').never()
flexmock(module.os).should_receive('environ').and_return(
{'USER': 'root', 'MYSQL_PWD': 'trustsome1'}
)
flexmock(module).should_receive('execute_command_with_processes').with_args(
('mariadb', '--batch', '--user', 'root'),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
).once()
module.restore_data_source_dump(
hook_config,
{},
data_source=hook_config[0],
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
borgmatic_runtime_directory='/run/borgmatic',
)
def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore():
hook_config = [
{
tests/unit/hooks/data_source/test_mysql.py
+245 -54
View File
@@ -60,6 +60,42 @@ def test_database_names_to_dump_queries_mysql_for_database_names():
assert names == ('foo', 'bar')
def test_database_names_to_dump_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).never()
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'mysql',
'--user',
'root',
'--skip-column-names',
'--batch',
'--execute',
'show schemas',
),
environment=environment,
).and_return('foo\nbar\nmysql\n').once()
names = module.database_names_to_dump(
{'name': 'all', 'password_transport': 'environment'},
{},
'root',
'trustsome1',
environment,
dry_run=False,
)
assert names == ('foo', 'bar')
def test_database_names_to_dump_runs_mysql_with_tls():
environment = flexmock()
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
@@ -141,9 +177,22 @@ def test_dump_data_sources_dumps_each_database():
'resolve_credential'
).and_return(None)
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
('bar',)
)
flexmock(module).should_receive('database_names_to_dump').with_args(
database=databases[0],
config={},
username=None,
password=None,
environment={'USER': 'root'},
dry_run=False,
).and_return(('foo',))
flexmock(module).should_receive('database_names_to_dump').with_args(
database=databases[1],
config={},
username=None,
password=None,
environment={'USER': 'root'},
dry_run=False,
).and_return(('bar',))
for name, process in zip(('foo', 'bar'), processes):
flexmock(module).should_receive('execute_dump_command').with_args(
@@ -179,9 +228,14 @@ def test_dump_data_sources_dumps_with_password():
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
('bar',)
)
flexmock(module).should_receive('database_names_to_dump').with_args(
database=database,
config={},
username='root',
password='trustsome1',
environment={'USER': 'root'},
dry_run=False,
).and_return(('foo',))
flexmock(module).should_receive('execute_dump_command').with_args(
database=database,
@@ -205,6 +259,50 @@ def test_dump_data_sources_dumps_with_password():
) == [process]
def test_dump_data_sources_dumps_with_environment_password_transport_passes_password_environment_variable():
database = {
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'password_transport': 'environment',
}
process = flexmock()
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module).should_receive('database_names_to_dump').with_args(
database=database,
config={},
username='root',
password='trustsome1',
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
dry_run=False,
).and_return(('foo',))
flexmock(module).should_receive('execute_dump_command').with_args(
database=database,
config={},
username='root',
password='trustsome1',
dump_path=object,
database_names=('foo',),
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
dry_run=object,
dry_run_label=object,
).and_return(process).once()
assert module.dump_data_sources(
[database],
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=False,
) == [process]
def test_dump_data_sources_dumps_all_databases_at_once():
databases = [{'name': 'all'}]
process = flexmock()
@@ -272,6 +370,54 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
)
def test_dump_data_sources_errors_for_missing_all_databases():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
with pytest.raises(ValueError):
assert module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=False,
)
def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
assert (
module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=True,
)
== []
)
def test_database_names_to_dump_runs_mysql_with_list_options():
database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf --skip-ssl'}
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
@@ -375,6 +521,52 @@ def test_execute_dump_command_runs_mysqldump():
)
def test_execute_dump_command_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).never()
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module).should_receive('execute_command').with_args(
(
'mysqldump',
'--add-drop-database',
'--user',
'root',
'--databases',
'foo',
'--result-file',
'dump',
),
environment=None,
run_to_completion=False,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'foo', 'password_transport': 'environment'},
config={},
username='root',
password='trustsome1',
dump_path=flexmock(),
database_names=('foo',),
environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_runs_mysqldump_without_add_drop_database():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -761,54 +953,6 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump():
)
def test_dump_data_sources_errors_for_missing_all_databases():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
with pytest.raises(ValueError):
assert module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=False,
)
def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run():
databases = [{'name': 'all'}]
flexmock(module).should_receive('make_dump_path').and_return('')
flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
'databases/localhost/all'
)
flexmock(module).should_receive('database_names_to_dump').and_return(())
assert (
module.dump_data_sources(
databases,
{},
config_paths=('test.yaml',),
borgmatic_runtime_directory='/run/borgmatic',
patterns=[],
dry_run=True,
)
== []
)
def test_restore_data_source_dump_runs_mysql_to_restore():
hook_config = [{'name': 'foo'}, {'name': 'bar'}]
extract_process = flexmock(stdout=flexmock())
@@ -1092,6 +1236,53 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password():
)
def test_restore_data_source_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
hook_config = [
{
'name': 'foo',
'username': 'root',
'password': 'trustsome1',
'password_transport': 'environment',
}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential'
).replace_with(lambda value, config: value)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'parse_extra_options'
).and_return((), None)
flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
'make_defaults_file_options'
).never()
flexmock(module.os).should_receive('environ').and_return(
{'USER': 'root', 'MYSQL_PWD': 'trustsome1'}
)
flexmock(module).should_receive('execute_command_with_processes').with_args(
('mysql', '--batch', '--user', 'root'),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
).once()
module.restore_data_source_dump(
hook_config,
{},
data_source=hook_config[0],
dry_run=False,
extract_process=extract_process,
connection_params={
'hostname': None,
'port': None,
'username': None,
'password': None,
},
borgmatic_runtime_directory='/run/borgmatic',
)
def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore():
hook_config = [
{