From c84815bfb040a8ed110a8eb293ffa93be7ec6af1 Mon Sep 17 00:00:00 2001 From: Nish_ <120EE0980@nitrkl.ac.in> Date: Sat, 15 Mar 2025 22:16:57 +0530 Subject: [PATCH 1/4] add custom dump and restore commands for sqlite hook Signed-off-by: Nish_ <120EE0980@nitrkl.ac.in> --- borgmatic/config/schema.yaml | 18 +++ borgmatic/hooks/data_source/sqlite.py | 16 ++- tests/unit/hooks/data_source/test_sqlite.py | 146 ++++++++++++++++++++ 3 files changed, 174 insertions(+), 6 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 13ab0686..dbd79d0c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1642,6 +1642,24 @@ properties: Path to the SQLite database file to restore to. Defaults to the "path" option. example: /var/lib/sqlite/users.db + sqlite_command: + type: string + description: | + Command to use instead of "sqlite3". This can be used + to run a specific sqlite3 version (e.g., one inside + a running container). If you run it from within + a container, make sure to mount your host's + ".borgmatic" folder into the container using the same + directory structure. Defaults to "sqlite3". + example: docker exec sqlite_container sqlite3 + sqlite_restore_command: + type: string + description: | + Command to run when restoring a database instead + of "sqlite3". This can be used to run a specific + sqlite3 version (e.g., one inside a running container). + Defaults to "sqlite3". + example: docker exec sqlite_container sqlite3 mongodb_databases: type: array items: diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 8c4b6008..c7ed267b 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -79,13 +79,17 @@ def dump_data_sources( ) continue - command = ( - 'sqlite3', + sqlite_command = tuple( + shlex.quote(part) + for part in shlex.split(database.get('sqlite_command') or 'sqlite3') + ) + command = sqlite_command + ( shlex.quote(database_path), '.dump', '>', shlex.quote(dump_filename), ) + logger.debug( f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' ) @@ -168,11 +172,11 @@ def restore_data_source_dump( except FileNotFoundError: # pragma: no cover pass - restore_command = ( - 'sqlite3', - database_path, + sqlite_restore_command = tuple( + shlex.quote(part) + for part in shlex.split(data_source.get('sqlite_restore_command') or 'sqlite3') ) - + restore_command = sqlite_restore_command + (shlex.quote(database_path),) # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. execute_command_with_processes( diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 6c5a5639..b54af4ef 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -116,6 +116,51 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): ) +def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_gets_escaped(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) + databases = [ + { + 'path': '/path/to/database1; naughty-command', + 'name': 'database1', + 'sqlite_command': 'custom_sqlite *', + }, + ] + processes = [flexmock()] + + flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic') + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + '/run/borgmatic/database' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + flexmock(module).should_receive('execute_command').with_args( + ( + 'custom_sqlite', # custom sqlite command + "'*'", # Should get shell escaped to prevent injection attacks. + "'/path/to/database1; naughty-command'", + '.dump', + '>', + '/run/borgmatic/database', + ), + shell=True, + run_to_completion=False, + ).and_return(processes[0]) + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + == processes + ) + + def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( flexmock() @@ -234,6 +279,41 @@ def test_restore_data_source_dump_restores_database(): ) +def test_restore_data_source_dump_runs_non_default_sqlite_restores_database(): + hook_config = [ + { + 'path': '/path/to/database', + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite *', + }, + {'name': 'other'}, + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + "'*'", # Should get shell escaped to prevent injection attacks. + '/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore(): hook_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} @@ -263,6 +343,38 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ) +def test_restore_data_source_dump_runs_non_default_sqlite_with_connection_params_uses_connection_params_for_restore(): + hook_config = [ + {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + 'cli/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source={ + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite', + }, + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': 'cli/path/to/database'}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore(): hook_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} @@ -292,6 +404,40 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ) +def test_restore_data_source_dump_runs_non_default_sqlite_without_connection_params_uses_restore_params_in_config_for_restore(): + hook_config = [ + { + 'path': '/path/to/database', + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite', + 'restore_path': 'config/path/to/database', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + 'config/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_does_not_restore_database_if_dry_run(): hook_config = [{'path': '/path/to/database', 'name': 'database'}] extract_process = flexmock(stdout=flexmock()) From f9612cc685b47748010eec787bf0bb7434b09467 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Mar 2025 21:37:23 -0700 Subject: [PATCH 2/4] Add SQLite custom command option to NEWS (#836). --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index b3d053fb..ef18be2b 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + * #836: Add a custom command option for the SQLite hook. * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers From 1d5713c4c5b645d9dd69ce4c41ada6949852b7b8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Mar 2025 21:42:45 -0700 Subject: [PATCH 3/4] Updated outdated schema comment referencing ~/.borgmatic path (#836). --- borgmatic/config/schema.yaml | 43 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index dbd79d0c..61c0c5e9 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1267,11 +1267,11 @@ properties: Command to use instead of "pg_dump" or "pg_dumpall". This can be used to run a specific pg_dump version (e.g., one inside a running container). If you run it - from within a container, make sure to mount your - host's ".borgmatic" folder into the container using - the same directory structure. Defaults to "pg_dump" - for single database dump or "pg_dumpall" to dump all - databases. + from within a container, make sure to mount the path in + the "user_runtime_directory" option from the host into + the container at the same location. Defaults to + "pg_dump" for single database dump or "pg_dumpall" to + dump all databases. example: docker exec my_pg_container pg_dump pg_restore_command: type: string @@ -1408,10 +1408,11 @@ properties: description: | Command to use instead of "mariadb-dump". This can be used to run a specific mariadb_dump version (e.g., one - inside a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "mariadb-dump". + inside a running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to + "mariadb-dump". example: docker exec mariadb_container mariadb-dump mariadb_command: type: string @@ -1550,12 +1551,12 @@ properties: mysql_dump_command: type: string description: | - Command to use instead of "mysqldump". This can be - used to run a specific mysql_dump version (e.g., one - inside a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "mysqldump". + Command to use instead of "mysqldump". This can be used + to run a specific mysql_dump version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to "mysqldump". example: docker exec mysql_container mysqldump mysql_command: type: string @@ -1645,12 +1646,12 @@ properties: sqlite_command: type: string description: | - Command to use instead of "sqlite3". This can be used - to run a specific sqlite3 version (e.g., one inside - a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "sqlite3". + Command to use instead of "sqlite3". This can be used to + run a specific sqlite3 version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to "sqlite3". example: docker exec sqlite_container sqlite3 sqlite_restore_command: type: string From c926f0bd5d75095ac4b70852279c38b68d9a7d9c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:31:34 -0700 Subject: [PATCH 4/4] Clarify documentation for `dump_data_sources` command hook (#790). --- docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 578601e2..8001a4c3 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -78,7 +78,8 @@ commands: This command hook has the following options: - * `before` or `after`: `dump_data_sources` + * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after: + * `dump_data_sources` runs before or after data sources are dumped (databases dumped or filesystems snapshotted) for each hook named in `hooks`. * `hooks`: Names of other hooks that this command hook applies to, e.g. `postgresql`, `mariadb`, `zfs`, `btrfs`, etc. Defaults to all hooks of the relevant type. * `run`: One or more shell commands or scripts to run when this command hook is triggered.