peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-22 02:03:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

For MariaDB/MySQL, dump events, routines, and tablespaces. For MariaDB, dump a subset of system data when dumping "mysql"/"all" (#1303).

Browse Source
This commit is contained in:
Dan Helfman
2026-05-08 09:38:13 -07:00
parent cce679248f
commit 411685280c
10 changed files with 192 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS
+5
View File
@@ -3,6 +3,11 @@
"working_directory".
* #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes
and globs.
* #1303: For the MariaDB and MySQL database hooks, include events, routines, and tablespaces when
dumping a database.
* #1303: For the MariaDB hook, include only a subset of system data when dumping the "mysql" system
database (or "all" databases), so the dump is actually restorable. See the documentation for more
information: https://torsion.org/borgmatic/reference/configuration/data-sources/mariadb/
* Update the KeePassXC credential hook to support KeePassXC's secret service integration. See the
documentation for more information:
https://torsion.org/borgmatic/reference/configuration/credentials/keepassxc/
borgmatic/hooks/data_source/mariadb.py
+15 -9
View File
@@ -1,4 +1,3 @@
import copy
import logging
import os
import re
@@ -108,6 +107,10 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file
return (f'--defaults-extra-file=/dev/fd/{read_file_descriptor}',)
EXCLUDED_SYSTEM_DATABASE_NAMES = ('information_schema', 'performance_schema', 'sys')
SYSTEM_DATABASE_NAME = 'mysql'
def database_names_to_dump(database, config, username, password, environment, dry_run):
'''
Given a requested database config, a configuration dict, a database username and password, an
@@ -166,17 +169,18 @@ def database_names_to_dump(database, config, username, password, environment, dr
working_directory=borgmatic.config.paths.get_working_directory(config),
)
# Dumping system databases directly doesn't work; too much gets dumped (including virtual tables
# that MariaDB recreates on startup) and so the dump isn't restorable. Therefore several system
# databases are excluded here. But also see below where select system tables from the "mysql"
# system table are dumped via the "--system" flag.
return tuple(
show_name.strip()
for show_name in show_lines
if show_name not in SYSTEM_DATABASE_NAMES
if show_name not in EXCLUDED_SYSTEM_DATABASE_NAMES
if not skip_names or show_name not in skip_names
)
SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
def execute_dump_command(
database,
config,
@@ -237,8 +241,9 @@ def execute_dump_command(
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--databases',)
+ database_names
+ ('--databases', '--events', '--routines', '--all-tablespaces')
+ (('--system=users,udfs,servers',) if SYSTEM_DATABASE_NAME in database_names else ())
+ tuple(name for name in database_names if name != SYSTEM_DATABASE_NAME)
+ ('--result-file', dump_filename)
)
@@ -325,6 +330,7 @@ def dump_data_sources(
raise ValueError('Cannot find any MariaDB databases to dump.')
# Database dumps to individual files.
if database['name'] == 'all' and database.get('format'):
for database_name in dump_database_names:
dumps_metadata.append(
@@ -337,8 +343,7 @@ def dump_data_sources(
database.get('container'),
)
)
renamed_database = copy.copy(database)
renamed_database['name'] = database_name
renamed_database = dict(database, name=database_name)
processes.append(
execute_dump_command(
renamed_database,
@@ -352,6 +357,7 @@ def dump_data_sources(
dry_run_label,
),
)
# Database dumps all to one file.
else:
dumps_metadata.append(
borgmatic.actions.restore.Dump(
borgmatic/hooks/data_source/mysql.py
+2 -4
View File
@@ -1,4 +1,3 @@
import copy
import logging
import os
import shlex
@@ -166,7 +165,7 @@ def execute_dump_command(
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
+ ('--databases',)
+ ('--databases', '--events', '--routines', '--all-tablespaces')
+ database_names
+ ('--result-file', dump_filename)
)
@@ -266,8 +265,7 @@ def dump_data_sources(
database.get('container'),
)
)
renamed_database = copy.copy(database)
renamed_database['name'] = database_name
renamed_database = dict(database, name=database_name)
processes.append(
execute_dump_command(
renamed_database,
docs/reference/configuration/data-sources/mariadb.md
+37 -1
View File
@@ -16,10 +16,46 @@ mariadb_databases:
```
### Full configuration
## System databases
<span class="minilink minilink-addedin">New in version 2.1.6</span> When dumping
["all"
databases](https://torsion.org/borgmatic/how-to/backup-your-databases/#all-databases),
borgmatic excludes most data coming from [MariaDB system
databases](https://mariadb.com/docs/server/reference/system-tables),
because much of it is populated on MariaDB startup and thus not restorable (or
just unnecessary to backup). The system data that borgmatic does include in
these dumps are: users, roles, grants, user-defined functions, and remote
servers.
Within a Borg archive, you can find this data stored in a dump named `mysql`—the
name of the system table this data comes from. And if you'd like to dump this
data without having to dump "all" databases, then you can configure a database
named `mysql` in your borgmatic configuration. For example:
```yaml
mariadb_databases:
- name: mysql
```
Even in this case though, only the subset of system data described above is
included in the dump.
<span class="minilink minilink-addedin">Prior to version 2.1.6</span> Dumps of
"all" databases excluded system databases and all of their data. Additionally,
explicitly dumping `mysql` was treated like any other database—and thus wasn't
easily restorable.
## Full configuration
{% include snippet/configuration/sample.md %}
```yaml
{% include borgmatic/mariadb_databases.yaml %}
```
## Related documentation
* [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
docs/reference/configuration/data-sources/mongodb.md
+6 -1
View File
@@ -14,10 +14,15 @@ mongodb_databases:
```
### Full configuration
## Full configuration
{% include snippet/configuration/sample.md %}
```yaml
{% include borgmatic/mongodb_databases.yaml %}
```
## Related documentation
* [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
docs/reference/configuration/data-sources/mysql.md
+5
View File
@@ -21,3 +21,8 @@ mysql_databases:
```yaml
{% include borgmatic/mysql_databases.yaml %}
```
## Related documentation
* [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
docs/reference/configuration/data-sources/postgresql.md
+5
View File
@@ -76,3 +76,8 @@ roles](https://www.postgresql.org/docs/current/predefined-roles.html) and
```yaml
{% include borgmatic/postgresql_databases.yaml %}
```
## Related documentation
* [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
docs/reference/configuration/data-sources/sqlite.md
+5
View File
@@ -22,3 +22,8 @@ sqlite_databases:
```yaml
{% include borgmatic/sqlite_databases.yaml %}
```
## Related documentation
* [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
tests/unit/hooks/data_source/test_mariadb.py
+85 -6
View File
@@ -218,7 +218,7 @@ def test_database_names_to_dump_queries_mariadb_for_database_names():
),
environment=environment,
working_directory='/path/to/working/dir',
).and_yield('foo', 'bar', 'mysql').once()
).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all'},
@@ -256,7 +256,7 @@ def test_database_names_to_dump_with_database_name_all_and_skip_names_filters_ou
),
environment=environment,
working_directory=None,
).and_yield('foo', 'bar', 'baz', 'mysql').once()
).and_yield('foo', 'bar', 'baz').once()
names = module.database_names_to_dump(
{'name': 'all', 'skip_names': ('foo', 'bar')},
@@ -296,7 +296,7 @@ def test_database_names_to_dump_runs_mariadb_with_socket_path():
),
environment=environment,
working_directory=None,
).and_yield('foo', 'bar', 'mysql').once()
).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'socket_path': '/socket'},
@@ -331,7 +331,7 @@ def test_database_names_to_dump_with_environment_password_transport_skips_defaul
),
environment=environment,
working_directory=None,
).and_yield('foo', 'bar', 'mysql').once()
).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'password_transport': 'environment'},
@@ -370,7 +370,7 @@ def test_database_names_to_dump_runs_mariadb_with_tls():
),
environment=environment,
working_directory=None,
).and_yield('foo', 'bar', 'mysql').once()
).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': True},
@@ -409,7 +409,7 @@ def test_database_names_to_dump_runs_mariadb_without_tls():
),
environment=environment,
working_directory=None,
).and_yield('foo', 'bar', 'mysql').once()
).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': False},
@@ -856,6 +856,9 @@ def test_execute_dump_command_runs_mariadb_dump():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -881,6 +884,58 @@ def test_execute_dump_command_runs_mariadb_dump():
)
def test_execute_dump_command_substitutes_system_flag_for_system_database_name():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential',
).replace_with(lambda value, config: value)
flexmock(module).should_receive('parse_extra_options').and_return((), None)
flexmock(module.database_config).should_receive('resolve_database_option').and_return(None)
flexmock(module).should_receive('make_defaults_file_options').with_args(
'root',
'trustsome1',
None,
).and_return(('--defaults-extra-file=/dev/fd/99',))
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module).should_receive('execute_command').with_args(
(
'mariadb-dump',
'--defaults-extra-file=/dev/fd/99',
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'--system=users,udfs,servers',
'--result-file',
'dump',
),
environment=None,
run_to_completion=False,
working_directory=None,
).and_return(process).once()
assert (
module.execute_dump_command(
database={'name': 'mysql'},
config={},
username='root',
password='trustsome1',
dump_path=flexmock(),
database_names=('mysql',),
environment=None,
dry_run=False,
dry_run_label='',
)
== process
)
def test_execute_dump_command_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -902,6 +957,9 @@ def test_execute_dump_command_with_environment_password_transport_skips_defaults
'--user',
'root',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -950,6 +1008,9 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database():
'--defaults-extra-file=/dev/fd/99',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1007,6 +1068,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port():
'--protocol',
'tcp',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1057,6 +1121,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_tls():
'--single-transaction',
'--ssl',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1107,6 +1174,9 @@ def test_execute_dump_command_runs_mariadb_dump_without_tls():
'--single-transaction',
'--skip-ssl',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1156,6 +1226,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1206,6 +1279,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_options():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1256,6 +1332,9 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
tests/unit/hooks/data_source/test_mysql.py
+27
View File
@@ -719,6 +719,9 @@ def test_execute_dump_command_runs_mysqldump():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -769,6 +772,9 @@ def test_execute_dump_command_with_environment_password_transport_skips_defaults
'--user',
'root',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -817,6 +823,9 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database():
'--defaults-extra-file=/dev/fd/99',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -874,6 +883,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port():
'--protocol',
'tcp',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -924,6 +936,9 @@ def test_execute_dump_command_runs_mysqldump_with_tls():
'--single-transaction',
'--ssl',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -974,6 +989,9 @@ def test_execute_dump_command_runs_mysqldump_without_tls():
'--single-transaction',
'--skip-ssl',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1023,6 +1041,9 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1073,6 +1094,9 @@ def test_execute_dump_command_runs_mysqldump_with_options():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1122,6 +1146,9 @@ def test_execute_dump_command_runs_non_default_mysqldump():
'--add-drop-database',
'--single-transaction',
'--databases',
'--events',
'--routines',
'--all-tablespaces',
'foo',
'--result-file',
'dump',