diff --git a/NEWS b/NEWS
index da578889..bee04626 100644
--- a/NEWS
+++ b/NEWS
@@ -3,6 +3,11 @@
"working_directory".
* #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes
and globs.
+ * #1303: For the MariaDB and MySQL database hooks, include events, routines, and tablespaces when
+ dumping a database.
+ * #1303: For the MariaDB hook, include only a subset of system data when dumping the "mysql" system
+ database (or "all" databases), so the dump is actually restorable. See the documentation for more
+ information: https://torsion.org/borgmatic/reference/configuration/data-sources/mariadb/
* Update the KeePassXC credential hook to support KeePassXC's secret service integration. See the
documentation for more information:
https://torsion.org/borgmatic/reference/configuration/credentials/keepassxc/
diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py
index 352afdde..f83bcc8e 100644
--- a/borgmatic/hooks/data_source/mariadb.py
+++ b/borgmatic/hooks/data_source/mariadb.py
@@ -1,4 +1,3 @@
-import copy
import logging
import os
import re
@@ -108,6 +107,10 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file
return (f'--defaults-extra-file=/dev/fd/{read_file_descriptor}',)
+EXCLUDED_SYSTEM_DATABASE_NAMES = ('information_schema', 'performance_schema', 'sys')
+SYSTEM_DATABASE_NAME = 'mysql'
+
+
def database_names_to_dump(database, config, username, password, environment, dry_run):
'''
Given a requested database config, a configuration dict, a database username and password, an
@@ -166,17 +169,18 @@ def database_names_to_dump(database, config, username, password, environment, dr
working_directory=borgmatic.config.paths.get_working_directory(config),
)
+ # Dumping system databases directly doesn't work; too much gets dumped (including virtual tables
+ # that MariaDB recreates on startup) and so the dump isn't restorable. Therefore several system
+ # databases are excluded here. But also see below where select system tables from the "mysql"
+ # system table are dumped via the "--system" flag.
return tuple(
show_name.strip()
for show_name in show_lines
- if show_name not in SYSTEM_DATABASE_NAMES
+ if show_name not in EXCLUDED_SYSTEM_DATABASE_NAMES
if not skip_names or show_name not in skip_names
)
-SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
-
-
def execute_dump_command(
database,
config,
@@ -237,8 +241,9 @@ def execute_dump_command(
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
- + ('--databases',)
- + database_names
+ + ('--databases', '--events', '--routines', '--all-tablespaces')
+ + (('--system=users,udfs,servers',) if SYSTEM_DATABASE_NAME in database_names else ())
+ + tuple(name for name in database_names if name != SYSTEM_DATABASE_NAME)
+ ('--result-file', dump_filename)
)
@@ -325,6 +330,7 @@ def dump_data_sources(
raise ValueError('Cannot find any MariaDB databases to dump.')
+ # Database dumps to individual files.
if database['name'] == 'all' and database.get('format'):
for database_name in dump_database_names:
dumps_metadata.append(
@@ -337,8 +343,7 @@ def dump_data_sources(
database.get('container'),
)
)
- renamed_database = copy.copy(database)
- renamed_database['name'] = database_name
+ renamed_database = dict(database, name=database_name)
processes.append(
execute_dump_command(
renamed_database,
@@ -352,6 +357,7 @@ def dump_data_sources(
dry_run_label,
),
)
+ # Database dumps all to one file.
else:
dumps_metadata.append(
borgmatic.actions.restore.Dump(
diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py
index c375cd18..af17eb62 100644
--- a/borgmatic/hooks/data_source/mysql.py
+++ b/borgmatic/hooks/data_source/mysql.py
@@ -1,4 +1,3 @@
-import copy
import logging
import os
import shlex
@@ -166,7 +165,7 @@ def execute_dump_command(
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
- + ('--databases',)
+ + ('--databases', '--events', '--routines', '--all-tablespaces')
+ database_names
+ ('--result-file', dump_filename)
)
@@ -266,8 +265,7 @@ def dump_data_sources(
database.get('container'),
)
)
- renamed_database = copy.copy(database)
- renamed_database['name'] = database_name
+ renamed_database = dict(database, name=database_name)
processes.append(
execute_dump_command(
renamed_database,
diff --git a/docs/reference/configuration/data-sources/mariadb.md b/docs/reference/configuration/data-sources/mariadb.md
index e18ce23f..559819bf 100644
--- a/docs/reference/configuration/data-sources/mariadb.md
+++ b/docs/reference/configuration/data-sources/mariadb.md
@@ -16,10 +16,46 @@ mariadb_databases:
```
-### Full configuration
+## System databases
+
+New in version 2.1.6 When dumping
+["all"
+databases](https://torsion.org/borgmatic/how-to/backup-your-databases/#all-databases),
+borgmatic excludes most data coming from [MariaDB system
+databases](https://mariadb.com/docs/server/reference/system-tables),
+because much of it is populated on MariaDB startup and thus not restorable (or
+just unnecessary to backup). The system data that borgmatic does include in
+these dumps are: users, roles, grants, user-defined functions, and remote
+servers.
+
+Within a Borg archive, you can find this data stored in a dump named `mysql`—the
+name of the system table this data comes from. And if you'd like to dump this
+data without having to dump "all" databases, then you can configure a database
+named `mysql` in your borgmatic configuration. For example:
+
+```yaml
+mariadb_databases:
+ - name: mysql
+```
+
+Even in this case though, only the subset of system data described above is
+included in the dump.
+
+Prior to version 2.1.6 Dumps of
+"all" databases excluded system databases and all of their data. Additionally,
+explicitly dumping `mysql` was treated like any other database—and thus wasn't
+easily restorable.
+
+
+## Full configuration
{% include snippet/configuration/sample.md %}
```yaml
{% include borgmatic/mariadb_databases.yaml %}
```
+
+
+## Related documentation
+
+ * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
diff --git a/docs/reference/configuration/data-sources/mongodb.md b/docs/reference/configuration/data-sources/mongodb.md
index 4d7556b3..c81b388f 100644
--- a/docs/reference/configuration/data-sources/mongodb.md
+++ b/docs/reference/configuration/data-sources/mongodb.md
@@ -14,10 +14,15 @@ mongodb_databases:
```
-### Full configuration
+## Full configuration
{% include snippet/configuration/sample.md %}
```yaml
{% include borgmatic/mongodb_databases.yaml %}
```
+
+
+## Related documentation
+
+ * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
diff --git a/docs/reference/configuration/data-sources/mysql.md b/docs/reference/configuration/data-sources/mysql.md
index d9ee55a9..479af73d 100644
--- a/docs/reference/configuration/data-sources/mysql.md
+++ b/docs/reference/configuration/data-sources/mysql.md
@@ -21,3 +21,8 @@ mysql_databases:
```yaml
{% include borgmatic/mysql_databases.yaml %}
```
+
+
+## Related documentation
+
+ * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
diff --git a/docs/reference/configuration/data-sources/postgresql.md b/docs/reference/configuration/data-sources/postgresql.md
index dbc71a2b..8f1999c9 100644
--- a/docs/reference/configuration/data-sources/postgresql.md
+++ b/docs/reference/configuration/data-sources/postgresql.md
@@ -76,3 +76,8 @@ roles](https://www.postgresql.org/docs/current/predefined-roles.html) and
```yaml
{% include borgmatic/postgresql_databases.yaml %}
```
+
+
+## Related documentation
+
+ * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
diff --git a/docs/reference/configuration/data-sources/sqlite.md b/docs/reference/configuration/data-sources/sqlite.md
index 9931ca2f..310e86cc 100644
--- a/docs/reference/configuration/data-sources/sqlite.md
+++ b/docs/reference/configuration/data-sources/sqlite.md
@@ -22,3 +22,8 @@ sqlite_databases:
```yaml
{% include borgmatic/sqlite_databases.yaml %}
```
+
+
+## Related documentation
+
+ * [How to backup your databases](https://torsion.org/borgmatic/how-to/backup-your-databases/)
diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py
index 04d4498f..c0af5485 100644
--- a/tests/unit/hooks/data_source/test_mariadb.py
+++ b/tests/unit/hooks/data_source/test_mariadb.py
@@ -218,7 +218,7 @@ def test_database_names_to_dump_queries_mariadb_for_database_names():
),
environment=environment,
working_directory='/path/to/working/dir',
- ).and_yield('foo', 'bar', 'mysql').once()
+ ).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all'},
@@ -256,7 +256,7 @@ def test_database_names_to_dump_with_database_name_all_and_skip_names_filters_ou
),
environment=environment,
working_directory=None,
- ).and_yield('foo', 'bar', 'baz', 'mysql').once()
+ ).and_yield('foo', 'bar', 'baz').once()
names = module.database_names_to_dump(
{'name': 'all', 'skip_names': ('foo', 'bar')},
@@ -296,7 +296,7 @@ def test_database_names_to_dump_runs_mariadb_with_socket_path():
),
environment=environment,
working_directory=None,
- ).and_yield('foo', 'bar', 'mysql').once()
+ ).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'socket_path': '/socket'},
@@ -331,7 +331,7 @@ def test_database_names_to_dump_with_environment_password_transport_skips_defaul
),
environment=environment,
working_directory=None,
- ).and_yield('foo', 'bar', 'mysql').once()
+ ).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'password_transport': 'environment'},
@@ -370,7 +370,7 @@ def test_database_names_to_dump_runs_mariadb_with_tls():
),
environment=environment,
working_directory=None,
- ).and_yield('foo', 'bar', 'mysql').once()
+ ).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': True},
@@ -409,7 +409,7 @@ def test_database_names_to_dump_runs_mariadb_without_tls():
),
environment=environment,
working_directory=None,
- ).and_yield('foo', 'bar', 'mysql').once()
+ ).and_yield('foo', 'bar').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': False},
@@ -856,6 +856,9 @@ def test_execute_dump_command_runs_mariadb_dump():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -881,6 +884,58 @@ def test_execute_dump_command_runs_mariadb_dump():
)
+def test_execute_dump_command_substitutes_system_flag_for_system_database_name():
+ process = flexmock()
+ flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
+ flexmock(module.os.path).should_receive('exists').and_return(False)
+ flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+ 'resolve_credential',
+ ).replace_with(lambda value, config: value)
+ flexmock(module).should_receive('parse_extra_options').and_return((), None)
+ flexmock(module.database_config).should_receive('resolve_database_option').and_return(None)
+ flexmock(module).should_receive('make_defaults_file_options').with_args(
+ 'root',
+ 'trustsome1',
+ None,
+ ).and_return(('--defaults-extra-file=/dev/fd/99',))
+ flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+ flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
+
+ flexmock(module).should_receive('execute_command').with_args(
+ (
+ 'mariadb-dump',
+ '--defaults-extra-file=/dev/fd/99',
+ '--add-drop-database',
+ '--single-transaction',
+ '--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
+ '--system=users,udfs,servers',
+ '--result-file',
+ 'dump',
+ ),
+ environment=None,
+ run_to_completion=False,
+ working_directory=None,
+ ).and_return(process).once()
+
+ assert (
+ module.execute_dump_command(
+ database={'name': 'mysql'},
+ config={},
+ username='root',
+ password='trustsome1',
+ dump_path=flexmock(),
+ database_names=('mysql',),
+ environment=None,
+ dry_run=False,
+ dry_run_label='',
+ )
+ == process
+ )
+
+
def test_execute_dump_command_with_environment_password_transport_skips_defaults_file_and_passes_user_flag():
process = flexmock()
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -902,6 +957,9 @@ def test_execute_dump_command_with_environment_password_transport_skips_defaults
'--user',
'root',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -950,6 +1008,9 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database():
'--defaults-extra-file=/dev/fd/99',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1007,6 +1068,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port():
'--protocol',
'tcp',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1057,6 +1121,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_tls():
'--single-transaction',
'--ssl',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1107,6 +1174,9 @@ def test_execute_dump_command_runs_mariadb_dump_without_tls():
'--single-transaction',
'--skip-ssl',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1156,6 +1226,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1206,6 +1279,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_options():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1256,6 +1332,9 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py
index 310ef3b2..e32ae653 100644
--- a/tests/unit/hooks/data_source/test_mysql.py
+++ b/tests/unit/hooks/data_source/test_mysql.py
@@ -719,6 +719,9 @@ def test_execute_dump_command_runs_mysqldump():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -769,6 +772,9 @@ def test_execute_dump_command_with_environment_password_transport_skips_defaults
'--user',
'root',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -817,6 +823,9 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database():
'--defaults-extra-file=/dev/fd/99',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -874,6 +883,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port():
'--protocol',
'tcp',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -924,6 +936,9 @@ def test_execute_dump_command_runs_mysqldump_with_tls():
'--single-transaction',
'--ssl',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -974,6 +989,9 @@ def test_execute_dump_command_runs_mysqldump_without_tls():
'--single-transaction',
'--skip-ssl',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1023,6 +1041,9 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1073,6 +1094,9 @@ def test_execute_dump_command_runs_mysqldump_with_options():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',
@@ -1122,6 +1146,9 @@ def test_execute_dump_command_runs_non_default_mysqldump():
'--add-drop-database',
'--single-transaction',
'--databases',
+ '--events',
+ '--routines',
+ '--all-tablespaces',
'foo',
'--result-file',
'dump',