From 411685280c46eb8696e3cbb6a4c50146f61f2b0b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 8 May 2026 09:38:13 -0700 Subject: [PATCH] For MariaDB/MySQL, dump events, routines, and tablespaces. For MariaDB, dump a subset of system data when dumping "mysql"/"all" (#1303). --- NEWS | 5 + borgmatic/hooks/data_source/mariadb.py | 24 +++-- borgmatic/hooks/data_source/mysql.py | 6 +- .../configuration/data-sources/mariadb.md | 38 +++++++- .../configuration/data-sources/mongodb.md | 7 +- .../configuration/data-sources/mysql.md | 5 + .../configuration/data-sources/postgresql.md | 5 + .../configuration/data-sources/sqlite.md | 5 + tests/unit/hooks/data_source/test_mariadb.py | 91 +++++++++++++++++-- tests/unit/hooks/data_source/test_mysql.py | 27 ++++++ 10 files changed, 192 insertions(+), 21 deletions(-) diff --git a/NEWS b/NEWS index da578889..bee04626 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,11 @@ "working_directory". * #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes and globs. + * #1303: For the MariaDB and MySQL database hooks, include events, routines, and tablespaces when + dumping a database. + * #1303: For the MariaDB hook, include only a subset of system data when dumping the "mysql" system + database (or "all" databases), so the dump is actually restorable. See the documentation for more + information: https://torsion.org/borgmatic/reference/configuration/data-sources/mariadb/ * Update the KeePassXC credential hook to support KeePassXC's secret service integration. See the documentation for more information: https://torsion.org/borgmatic/reference/configuration/credentials/keepassxc/ diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 352afdde..f83bcc8e 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -1,4 +1,3 @@ -import copy import logging import os import re @@ -108,6 +107,10 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file return (f'--defaults-extra-file=/dev/fd/{read_file_descriptor}',) +EXCLUDED_SYSTEM_DATABASE_NAMES = ('information_schema', 'performance_schema', 'sys') +SYSTEM_DATABASE_NAME = 'mysql' + + def database_names_to_dump(database, config, username, password, environment, dry_run): ''' Given a requested database config, a configuration dict, a database username and password, an @@ -166,17 +169,18 @@ def database_names_to_dump(database, config, username, password, environment, dr working_directory=borgmatic.config.paths.get_working_directory(config), ) + # Dumping system databases directly doesn't work; too much gets dumped (including virtual tables + # that MariaDB recreates on startup) and so the dump isn't restorable. Therefore several system + # databases are excluded here. But also see below where select system tables from the "mysql" + # system table are dumped via the "--system" flag. return tuple( show_name.strip() for show_name in show_lines - if show_name not in SYSTEM_DATABASE_NAMES + if show_name not in EXCLUDED_SYSTEM_DATABASE_NAMES if not skip_names or show_name not in skip_names ) -SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') - - def execute_dump_command( database, config, @@ -237,8 +241,9 @@ def execute_dump_command( + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) - + ('--databases',) - + database_names + + ('--databases', '--events', '--routines', '--all-tablespaces') + + (('--system=users,udfs,servers',) if SYSTEM_DATABASE_NAME in database_names else ()) + + tuple(name for name in database_names if name != SYSTEM_DATABASE_NAME) + ('--result-file', dump_filename) ) @@ -325,6 +330,7 @@ def dump_data_sources( raise ValueError('Cannot find any MariaDB databases to dump.') + # Database dumps to individual files. if database['name'] == 'all' and database.get('format'): for database_name in dump_database_names: dumps_metadata.append( @@ -337,8 +343,7 @@ def dump_data_sources( database.get('container'), ) ) - renamed_database = copy.copy(database) - renamed_database['name'] = database_name + renamed_database = dict(database, name=database_name) processes.append( execute_dump_command( renamed_database, @@ -352,6 +357,7 @@ def dump_data_sources( dry_run_label, ), ) + # Database dumps all to one file. else: dumps_metadata.append( borgmatic.actions.restore.Dump( diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index c375cd18..af17eb62 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -1,4 +1,3 @@ -import copy import logging import os import shlex @@ -166,7 +165,7 @@ def execute_dump_command( + (('--user', username) if username and password_transport == 'environment' else ()) + (('--ssl',) if database.get('tls') is True else ()) + (('--skip-ssl',) if database.get('tls') is False else ()) - + ('--databases',) + + ('--databases', '--events', '--routines', '--all-tablespaces') + database_names + ('--result-file', dump_filename) ) @@ -266,8 +265,7 @@ def dump_data_sources( database.get('container'), ) ) - renamed_database = copy.copy(database) - renamed_database['name'] = database_name + renamed_database = dict(database, name=database_name) processes.append( execute_dump_command( renamed_database, diff --git a/docs/reference/configuration/data-sources/mariadb.md b/docs/reference/configuration/data-sources/mariadb.md index e18ce23f..559819bf 100644 --- a/docs/reference/configuration/data-sources/mariadb.md +++ b/docs/reference/configuration/data-sources/mariadb.md @@ -16,10 +16,46 @@ mariadb_databases: ``` -### Full configuration +## System databases + +New in version 2.1.6 When dumping +["all" +databases](https://torsion.org/borgmatic/how-to/backup-your-databases/#all-databases), +borgmatic excludes most data coming from [MariaDB system +databases](https://mariadb.com/docs/server/reference/system-tables), +because much of it is populated on MariaDB startup and thus not restorable (or +just unnecessary to backup). The system data that borgmatic does include in +these dumps are: users, roles, grants, user-defined functions, and remote +servers. + +Within a Borg archive, you can find this data stored in a dump named `mysql`—the +name of the system table this data comes from. And if you'd like to dump this +data without having to dump "all" databases, then you can configure a database +named `mysql` in your borgmatic configuration. For example: + +```yaml +mariadb_databases: + - name: mysql +``` + +Even in this case though, only the subset of system data described above is +included in the dump. + +Prior to version 2.1.6 Dumps of +"all" databases excluded system databases and all of their data. Additionally, +explicitly dumping `mysql` was treated like any other database—and thus wasn't +easily restorable. + + +## Full configuration {% include snippet/configuration/sample.md %} ```yaml {% include borgmatic/mariadb_databases.yaml %} ``` + + +## Related documentation + + * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/) diff --git a/docs/reference/configuration/data-sources/mongodb.md b/docs/reference/configuration/data-sources/mongodb.md index 4d7556b3..c81b388f 100644 --- a/docs/reference/configuration/data-sources/mongodb.md +++ b/docs/reference/configuration/data-sources/mongodb.md @@ -14,10 +14,15 @@ mongodb_databases: ``` -### Full configuration +## Full configuration {% include snippet/configuration/sample.md %} ```yaml {% include borgmatic/mongodb_databases.yaml %} ``` + + +## Related documentation + + * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/) diff --git a/docs/reference/configuration/data-sources/mysql.md b/docs/reference/configuration/data-sources/mysql.md index d9ee55a9..479af73d 100644 --- a/docs/reference/configuration/data-sources/mysql.md +++ b/docs/reference/configuration/data-sources/mysql.md @@ -21,3 +21,8 @@ mysql_databases: ```yaml {% include borgmatic/mysql_databases.yaml %} ``` + + +## Related documentation + + * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/) diff --git a/docs/reference/configuration/data-sources/postgresql.md b/docs/reference/configuration/data-sources/postgresql.md index dbc71a2b..8f1999c9 100644 --- a/docs/reference/configuration/data-sources/postgresql.md +++ b/docs/reference/configuration/data-sources/postgresql.md @@ -76,3 +76,8 @@ roles](https://www.postgresql.org/docs/current/predefined-roles.html) and ```yaml {% include borgmatic/postgresql_databases.yaml %} ``` + + +## Related documentation + + * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/) diff --git a/docs/reference/configuration/data-sources/sqlite.md b/docs/reference/configuration/data-sources/sqlite.md index 9931ca2f..310e86cc 100644 --- a/docs/reference/configuration/data-sources/sqlite.md +++ b/docs/reference/configuration/data-sources/sqlite.md @@ -22,3 +22,8 @@ sqlite_databases: ```yaml {% include borgmatic/sqlite_databases.yaml %} ``` + + +## Related documentation + + * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 04d4498f..c0af5485 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -218,7 +218,7 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): ), environment=environment, working_directory='/path/to/working/dir', - ).and_yield('foo', 'bar', 'mysql').once() + ).and_yield('foo', 'bar').once() names = module.database_names_to_dump( {'name': 'all'}, @@ -256,7 +256,7 @@ def test_database_names_to_dump_with_database_name_all_and_skip_names_filters_ou ), environment=environment, working_directory=None, - ).and_yield('foo', 'bar', 'baz', 'mysql').once() + ).and_yield('foo', 'bar', 'baz').once() names = module.database_names_to_dump( {'name': 'all', 'skip_names': ('foo', 'bar')}, @@ -296,7 +296,7 @@ def test_database_names_to_dump_runs_mariadb_with_socket_path(): ), environment=environment, working_directory=None, - ).and_yield('foo', 'bar', 'mysql').once() + ).and_yield('foo', 'bar').once() names = module.database_names_to_dump( {'name': 'all', 'socket_path': '/socket'}, @@ -331,7 +331,7 @@ def test_database_names_to_dump_with_environment_password_transport_skips_defaul ), environment=environment, working_directory=None, - ).and_yield('foo', 'bar', 'mysql').once() + ).and_yield('foo', 'bar').once() names = module.database_names_to_dump( {'name': 'all', 'password_transport': 'environment'}, @@ -370,7 +370,7 @@ def test_database_names_to_dump_runs_mariadb_with_tls(): ), environment=environment, working_directory=None, - ).and_yield('foo', 'bar', 'mysql').once() + ).and_yield('foo', 'bar').once() names = module.database_names_to_dump( {'name': 'all', 'tls': True}, @@ -409,7 +409,7 @@ def test_database_names_to_dump_runs_mariadb_without_tls(): ), environment=environment, working_directory=None, - ).and_yield('foo', 'bar', 'mysql').once() + ).and_yield('foo', 'bar').once() names = module.database_names_to_dump( {'name': 'all', 'tls': False}, @@ -856,6 +856,9 @@ def test_execute_dump_command_runs_mariadb_dump(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -881,6 +884,58 @@ def test_execute_dump_command_runs_mariadb_dump(): ) +def test_execute_dump_command_substitutes_system_flag_for_system_database_name(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential', + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module.database_config).should_receive('resolve_database_option').and_return(None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', + 'trustsome1', + None, + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--single-transaction', + '--databases', + '--events', + '--routines', + '--all-tablespaces', + '--system=users,udfs,servers', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + working_directory=None, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'mysql'}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('mysql',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_with_environment_password_transport_skips_defaults_file_and_passes_user_flag(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -902,6 +957,9 @@ def test_execute_dump_command_with_environment_password_transport_skips_defaults '--user', 'root', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -950,6 +1008,9 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): '--defaults-extra-file=/dev/fd/99', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1007,6 +1068,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): '--protocol', 'tcp', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1057,6 +1121,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_tls(): '--single-transaction', '--ssl', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1107,6 +1174,9 @@ def test_execute_dump_command_runs_mariadb_dump_without_tls(): '--single-transaction', '--skip-ssl', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1156,6 +1226,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1206,6 +1279,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1256,6 +1332,9 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 310ef3b2..e32ae653 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -719,6 +719,9 @@ def test_execute_dump_command_runs_mysqldump(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -769,6 +772,9 @@ def test_execute_dump_command_with_environment_password_transport_skips_defaults '--user', 'root', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -817,6 +823,9 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): '--defaults-extra-file=/dev/fd/99', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -874,6 +883,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): '--protocol', 'tcp', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -924,6 +936,9 @@ def test_execute_dump_command_runs_mysqldump_with_tls(): '--single-transaction', '--ssl', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -974,6 +989,9 @@ def test_execute_dump_command_runs_mysqldump_without_tls(): '--single-transaction', '--skip-ssl', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1023,6 +1041,9 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1073,6 +1094,9 @@ def test_execute_dump_command_runs_mysqldump_with_options(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump', @@ -1122,6 +1146,9 @@ def test_execute_dump_command_runs_non_default_mysqldump(): '--add-drop-database', '--single-transaction', '--databases', + '--events', + '--routines', + '--all-tablespaces', 'foo', '--result-file', 'dump',